def test_get_layer_monitor_channels():
"""
Create a MLP with multiple layer types
and get layer monitoring channels for MLP.
"""
mlp = MLP(layers=[
FlattenerLayer(
CompositeLayer('composite',
[Linear(10, 'h0', 0.1),
Linear(10, 'h1', 0.1)], {
0: [1],
1: [0]
})),
Softmax(5, 'softmax', 0.1)
],
input_space=CompositeSpace([VectorSpace(15),
VectorSpace(20)]),
input_source=('features0', 'features1'))
dataset = VectorSpacesDataset(
(np.random.rand(20, 20).astype(theano.config.floatX),
np.random.rand(20, 15).astype(theano.config.floatX),
np.random.rand(20, 5).astype(theano.config.floatX)),
(CompositeSpace(
[VectorSpace(20), VectorSpace(15),
VectorSpace(5)]), ('features1', 'features0', 'targets')))
state_below = mlp.get_input_space().make_theano_batch()
targets = mlp.get_target_space().make_theano_batch()
mlp.get_layer_monitoring_channels(state_below=state_below,
state=None,
targets=targets)
def test_multiple_inputs():
"""
Create a VectorSpacesDataset with two inputs (features0 and features1)
and train an MLP which takes both inputs for 1 epoch.
"""
mlp = MLP(layers=[
FlattenerLayer(
CompositeLayer('composite',
[Linear(10, 'h0', 0.1),
Linear(10, 'h1', 0.1)], {
0: [1],
1: [0]
})),
Softmax(5, 'softmax', 0.1)
],
input_space=CompositeSpace([VectorSpace(15),
VectorSpace(20)]),
input_source=('features0', 'features1'))
dataset = VectorSpacesDataset(
(np.random.rand(20, 20).astype(theano.config.floatX),
np.random.rand(20, 15).astype(theano.config.floatX),
np.random.rand(20, 5).astype(theano.config.floatX)),
(CompositeSpace(
[VectorSpace(20), VectorSpace(15),
VectorSpace(5)]), ('features1', 'features0', 'targets')))
train = Train(dataset, mlp, SGD(0.1, batch_size=5))
train.algorithm.termination_criterion = EpochCounter(1)
train.main_loop()
def test_flattener_layer_state_separation_for_conv():
"""
Creates a CompositeLayer wrapping two Conv layers
and ensures that state gets correctly picked apart.
"""
conv1 = ConvElemwise(8, [2, 2], 'sf1', SigmoidConvNonlinearity(), .1)
conv2 = ConvElemwise(8, [2, 2], 'sf2', SigmoidConvNonlinearity(), .1)
mlp = MLP(layers=[FlattenerLayer(CompositeLayer('comp', [conv1, conv2]))],
input_space=Conv2DSpace(shape=[5, 5], num_channels=2))
topo_view = np.random.rand(10, 5, 5, 2).astype(theano.config.floatX)
y = np.random.rand(10, 256).astype(theano.config.floatX)
dataset = DenseDesignMatrix(topo_view=topo_view, y=y)
train = Train(dataset, mlp,
SGD(0.1, batch_size=5, monitoring_dataset=dataset))
train.algorithm.termination_criterion = EpochCounter(1)
train.main_loop()
def test_flattener_layer_state_separation_for_softmax():
"""
Creates a CompositeLayer wrapping two Softmax layers
and ensures that state gets correctly picked apart.
"""
soft1 = Softmax(5, 'sf1', .1)
soft2 = Softmax(5, 'sf2', .1)
mlp = MLP(layers=[FlattenerLayer(CompositeLayer('comp', [soft1, soft2]))],
nvis=2)
X = np.random.rand(20, 2).astype(theano.config.floatX)
y = np.random.rand(20, 10).astype(theano.config.floatX)
dataset = DenseDesignMatrix(X=X, y=y)
train = Train(dataset, mlp,
SGD(0.1, batch_size=5, monitoring_dataset=dataset))
train.algorithm.termination_criterion = EpochCounter(1)
train.main_loop()
def test_flattener_layer():
# To test the FlattenerLayer we create a very simple feed-forward neural
# network with two parallel linear layers. We then create two separate
# feed-forward neural networks with single linear layers. In principle,
# these two models should be identical if we start from the same
# parameters. This makes it easy to test that the composite layer works
# as expected.
# Create network with composite layers.
mlp_composite = MLP(layers=[
FlattenerLayer(
CompositeLayer('composite',
[Linear(2, 'h0', 0.1),
Linear(2, 'h1', 0.1)], {
0: [0],
1: [1]
}))
],
input_space=CompositeSpace(
[VectorSpace(5), VectorSpace(10)]),
input_source=('features0', 'features1'))
# Create network with single linear layer, corresponding to first
# layer in the composite network.
mlp_first_part = MLP(layers=[Linear(2, 'h0', 0.1)],
input_space=VectorSpace(5),
input_source=('features0'))
# Create network with single linear layer, corresponding to second
# layer in the composite network.
mlp_second_part = MLP(layers=[Linear(2, 'h1', 0.1)],
input_space=VectorSpace(10),
input_source=('features1'))
# Create dataset which we will test our networks against.
shared_dataset = np.random.rand(20, 19).astype(theano.config.floatX)
# Make dataset for composite network.
dataset_composite = VectorSpacesDataset(
(shared_dataset[:, 0:5], shared_dataset[:, 5:15],
shared_dataset[:, 15:19]), (CompositeSpace(
[VectorSpace(5), VectorSpace(10),
VectorSpace(4)]), ('features0', 'features1', 'targets')))
# Make dataset for first single linear layer network.
dataset_first_part = VectorSpacesDataset(
(shared_dataset[:, 0:5], shared_dataset[:, 15:17]),
(CompositeSpace([VectorSpace(5), VectorSpace(2)]),
('features0', 'targets')))
# Make dataset for second single linear layer network.
dataset_second_part = VectorSpacesDataset(
(shared_dataset[:, 5:15], shared_dataset[:, 17:19]),
(CompositeSpace([VectorSpace(10), VectorSpace(2)]),
('features1', 'targets')))
# Initialize all MLPs to start from zero weights.
mlp_composite.layers[0].raw_layer.layers[0].set_weights(
mlp_composite.layers[0].raw_layer.layers[0].get_weights() * 0.0)
mlp_composite.layers[0].raw_layer.layers[1].set_weights(
mlp_composite.layers[0].raw_layer.layers[1].get_weights() * 0.0)
mlp_first_part.layers[0].set_weights(
mlp_first_part.layers[0].get_weights() * 0.0)
mlp_second_part.layers[0].set_weights(
mlp_second_part.layers[0].get_weights() * 0.0)
# Train all models with their respective datasets.
train_composite = Train(dataset_composite, mlp_composite,
SGD(0.0001, batch_size=20))
train_composite.algorithm.termination_criterion = EpochCounter(1)
train_composite.main_loop()
train_first_part = Train(dataset_first_part, mlp_first_part,
SGD(0.0001, batch_size=20))
train_first_part.algorithm.termination_criterion = EpochCounter(1)
train_first_part.main_loop()
train_second_part = Train(dataset_second_part, mlp_second_part,
SGD(0.0001, batch_size=20))
train_second_part.algorithm.termination_criterion = EpochCounter(1)
train_second_part.main_loop()
# Check that the composite feed-forward neural network has learned
# same parameters as each individual feed-forward neural network.
np.testing.assert_allclose(
mlp_composite.layers[0].raw_layer.layers[0].get_weights(),
mlp_first_part.layers[0].get_weights())
np.testing.assert_allclose(
mlp_composite.layers[0].raw_layer.layers[1].get_weights(),
mlp_second_part.layers[0].get_weights())
# Check that we get same output given the same input on a randomly
# generated dataset.
X_composite = mlp_composite.get_input_space().make_theano_batch()
X_first_part = mlp_first_part.get_input_space().make_theano_batch()
X_second_part = mlp_second_part.get_input_space().make_theano_batch()
fprop_composite = theano.function(X_composite,
mlp_composite.fprop(X_composite))
fprop_first_part = theano.function([X_first_part],
mlp_first_part.fprop(X_first_part))
fprop_second_part = theano.function([X_second_part],
mlp_second_part.fprop(X_second_part))
X_data = np.random.random(size=(10, 15)).astype(theano.config.floatX)
y_data = np.random.randint(low=0, high=10, size=(10, 4))
np.testing.assert_allclose(
fprop_composite(X_data[:, 0:5], X_data[:, 5:15])[:, 0:2],
fprop_first_part(X_data[:, 0:5]))
np.testing.assert_allclose(
fprop_composite(X_data[:, 0:5], X_data[:, 5:15])[:, 2:4],
fprop_second_part(X_data[:, 5:15]))
# Finally check that calling the internal FlattenerLayer behaves
# as we would expect. First, retrieve the FlattenerLayer.
fl = mlp_composite.layers[0]
# Check that it agrees on the input space.
assert mlp_composite.get_input_space() == fl.get_input_space()
# Check that it agrees on the parameters.
for i in range(0, 4):
np.testing.assert_allclose(fl.get_params()[i].eval(),
mlp_composite.get_params()[i].eval())