def test_multiple_inputs():
"""
Create a VectorSpacesDataset with two inputs (features0 and features1)
and train an MLP which takes both inputs for 1 epoch.
"""
mlp = MLP(layers=[
FlattenerLayer(
CompositeLayer('composite',
[Linear(10, 'h0', 0.1),
Linear(10, 'h1', 0.1)], {
0: [1],
1: [0]
})),
Softmax(5, 'softmax', 0.1)
],
input_space=CompositeSpace([VectorSpace(15),
VectorSpace(20)]),
input_source=('features0', 'features1'))
dataset = VectorSpacesDataset(
(np.random.rand(20, 20).astype(theano.config.floatX),
np.random.rand(20, 15).astype(theano.config.floatX),
np.random.rand(20, 5).astype(theano.config.floatX)),
(CompositeSpace(
[VectorSpace(20), VectorSpace(15),
VectorSpace(5)]), ('features1', 'features0', 'targets')))
train = Train(dataset, mlp, SGD(0.1, batch_size=5))
train.algorithm.termination_criterion = EpochCounter(1)
train.main_loop()
def test_get_layer_monitor_channels():
"""
Create a MLP with multiple layer types
and get layer monitoring channels for MLP.
"""
mlp = MLP(layers=[
FlattenerLayer(
CompositeLayer('composite',
[Linear(10, 'h0', 0.1),
Linear(10, 'h1', 0.1)], {
0: [1],
1: [0]
})),
Softmax(5, 'softmax', 0.1)
],
input_space=CompositeSpace([VectorSpace(15),
VectorSpace(20)]),
input_source=('features0', 'features1'))
dataset = VectorSpacesDataset(
(np.random.rand(20, 20).astype(theano.config.floatX),
np.random.rand(20, 15).astype(theano.config.floatX),
np.random.rand(20, 5).astype(theano.config.floatX)),
(CompositeSpace(
[VectorSpace(20), VectorSpace(15),
VectorSpace(5)]), ('features1', 'features0', 'targets')))
state_below = mlp.get_input_space().make_theano_batch()
targets = mlp.get_target_space().make_theano_batch()
mlp.get_layer_monitoring_channels(state_below=state_below,
state=None,
targets=targets)
def test_input_and_target_source():
"""
Create a MLP and test input_source and target_source
for default and non-default options.
"""
mlp = MLP(
layers=[CompositeLayer(
'composite',
[Linear(10, 'h0', 0.1),
Linear(10, 'h1', 0.1)],
{
0: [1],
1: [0]
}
)
],
input_space=CompositeSpace([VectorSpace(15), VectorSpace(20)]),
input_source=('features0', 'features1'),
target_source=('targets0', 'targets1')
)
np.testing.assert_equal(mlp.get_input_source(), ('features0', 'features1'))
np.testing.assert_equal(mlp.get_target_source(), ('targets0', 'targets1'))
mlp = MLP(
layers=[Linear(10, 'h0', 0.1)],
input_space=VectorSpace(15)
)
np.testing.assert_equal(mlp.get_input_source(), 'features')
np.testing.assert_equal(mlp.get_target_source(), 'targets')
def _get_default_output_layer(self):
return CompositeLayer(layer_name='conditional',
layers=[
Linear(dim=self.ndim,
layer_name='mu',
irange=0.01),
Linear(dim=self.ndim,
layer_name='log_sigma',
irange=0.01)
])
def _get_default_output_layer(self):
return CompositeLayer(layer_name='composite',
layers=[
Linear(layer_name='1',
dim=self.ndim,
irange=0.01),
Linear(layer_name='2',
dim=self.ndim,
irange=0.01)
])
def test_nested_mlp():
"""
Constructs a nested MLP and tries to fprop through it
"""
inner_mlp = MLP(layers=[Linear(10, 'h0', 0.1), Linear(10, 'h1', 0.1)],
layer_name='inner_mlp')
outer_mlp = MLP(layers=[CompositeLayer(layer_name='composite',
layers=[inner_mlp,
Linear(10, 'h2', 0.1)])],
nvis=10)
X = outer_mlp.get_input_space().make_theano_batch()
f = theano.function([X], outer_mlp.fprop(X))
f(np.random.rand(5, 10).astype(theano.config.floatX))
def create_input_layer(self):
size = self.extender_dim * (self.ws * 2 + 1)
embed = Linear(layer_name='lin_embed', dim=size,
istdev=1. / sqrt(size))
return CompositeLayer(
layer_name='input',
layers=[
ProjectionLayer(layer_name='ext_words', dim=self.edim, irange=.1),
ProjectionLayer(layer_name='ext_feats', dim=self.fedim, irange=.1),
embed,
],
inputs_to_layers={0: [0], 1: [1], 2: [2]}
)
def create_input_layer(self):
return CompositeLayer(layer_name='input',
layers=[
ProjectionLayer(layer_name='words',
dim=self.edim,
irange=.1),
ProjectionLayer(layer_name='feats',
dim=self.fedim,
irange=.1),
],
inputs_to_layers={
0: [0],
1: [1]
})
def test_flattener_layer_state_separation_for_conv():
"""
Creates a CompositeLayer wrapping two Conv layers
and ensures that state gets correctly picked apart.
"""
conv1 = ConvElemwise(8, [2, 2], 'sf1', SigmoidConvNonlinearity(), .1)
conv2 = ConvElemwise(8, [2, 2], 'sf2', SigmoidConvNonlinearity(), .1)
mlp = MLP(layers=[FlattenerLayer(CompositeLayer('comp', [conv1, conv2]))],
input_space=Conv2DSpace(shape=[5, 5], num_channels=2))
topo_view = np.random.rand(10, 5, 5, 2).astype(theano.config.floatX)
y = np.random.rand(10, 256).astype(theano.config.floatX)
dataset = DenseDesignMatrix(topo_view=topo_view, y=y)
train = Train(dataset, mlp,
SGD(0.1, batch_size=5, monitoring_dataset=dataset))
train.algorithm.termination_criterion = EpochCounter(1)
train.main_loop()
def test_flattener_layer_state_separation_for_softmax():
"""
Creates a CompositeLayer wrapping two Softmax layers
and ensures that state gets correctly picked apart.
"""
soft1 = Softmax(5, 'sf1', .1)
soft2 = Softmax(5, 'sf2', .1)
mlp = MLP(layers=[FlattenerLayer(CompositeLayer('comp', [soft1, soft2]))],
nvis=2)
X = np.random.rand(20, 2).astype(theano.config.floatX)
y = np.random.rand(20, 10).astype(theano.config.floatX)
dataset = DenseDesignMatrix(X=X, y=y)
train = Train(dataset, mlp,
SGD(0.1, batch_size=5, monitoring_dataset=dataset))
train.algorithm.termination_criterion = EpochCounter(1)
train.main_loop()
def test_composite_layer():
"""
Test the routing functionality of the CompositeLayer
"""
# Without routing
composite_layer = CompositeLayer('composite_layer',
[Linear(2, 'h0', irange=0),
Linear(2, 'h1', irange=0),
Linear(2, 'h2', irange=0)])
mlp = MLP(nvis=2, layers=[composite_layer])
for i in range(3):
composite_layer.layers[i].set_weights(
np.eye(2, dtype=theano.config.floatX)
)
composite_layer.layers[i].set_biases(
np.zeros(2, dtype=theano.config.floatX)
)
X = tensor.matrix()
y = mlp.fprop(X)
funs = [theano.function([X], y_elem) for y_elem in y]
x_numeric = np.random.rand(2, 2).astype('float32')
y_numeric = [f(x_numeric) for f in funs]
assert np.all(x_numeric == y_numeric)
# With routing
for inputs_to_layers in [{0: [1], 1: [2], 2: [0]},
{0: [1], 1: [0, 2], 2: []},
{0: [], 1: []}]:
composite_layer = CompositeLayer('composite_layer',
[Linear(2, 'h0', irange=0),
Linear(2, 'h1', irange=0),
Linear(2, 'h2', irange=0)],
inputs_to_layers)
input_space = CompositeSpace([VectorSpace(dim=2),
VectorSpace(dim=2),
VectorSpace(dim=2)])
mlp = MLP(input_space=input_space, layers=[composite_layer])
for i in range(3):
composite_layer.layers[i].set_weights(
np.eye(2, dtype=theano.config.floatX)
)
composite_layer.layers[i].set_biases(
np.zeros(2, dtype=theano.config.floatX)
)
X = [tensor.matrix() for _ in range(3)]
y = mlp.fprop(X)
funs = [theano.function(X, y_elem, on_unused_input='ignore')
for y_elem in y]
x_numeric = [np.random.rand(2, 2).astype(theano.config.floatX)
for _ in range(3)]
y_numeric = [f(*x_numeric) for f in funs]
assert all([all([np.all(x_numeric[i] == y_numeric[j])
for j in inputs_to_layers[i]])
for i in inputs_to_layers])
# Get the weight decay expressions from a composite layer
composite_layer = CompositeLayer('composite_layer',
[Linear(2, 'h0', irange=0.1),
Linear(2, 'h1', irange=0.1)])
input_space = VectorSpace(dim=10)
mlp = MLP(input_space=input_space, layers=[composite_layer])
for attr, coeff in product(['get_weight_decay', 'get_l1_weight_decay'],
[[0.7, 0.3], 0.5]):
f = theano.function([], getattr(composite_layer, attr)(coeff))
if is_iterable(coeff):
g = theano.function(
[], tensor.sum([getattr(layer, attr)(c) for c, layer
in zip(coeff, composite_layer.layers)])
)
assert np.allclose(f(), g())
else:
g = theano.function(
[], tensor.sum([getattr(layer, attr)(coeff) for layer
in composite_layer.layers])
)
assert np.allclose(f(), g())
def test_flattener_layer():
# To test the FlattenerLayer we create a very simple feed-forward neural
# network with two parallel linear layers. We then create two separate
# feed-forward neural networks with single linear layers. In principle,
# these two models should be identical if we start from the same
# parameters. This makes it easy to test that the composite layer works
# as expected.
# Create network with composite layers.
mlp_composite = MLP(layers=[
FlattenerLayer(
CompositeLayer('composite',
[Linear(2, 'h0', 0.1),
Linear(2, 'h1', 0.1)], {
0: [0],
1: [1]
}))
],
input_space=CompositeSpace(
[VectorSpace(5), VectorSpace(10)]),
input_source=('features0', 'features1'))
# Create network with single linear layer, corresponding to first
# layer in the composite network.
mlp_first_part = MLP(layers=[Linear(2, 'h0', 0.1)],
input_space=VectorSpace(5),
input_source=('features0'))
# Create network with single linear layer, corresponding to second
# layer in the composite network.
mlp_second_part = MLP(layers=[Linear(2, 'h1', 0.1)],
input_space=VectorSpace(10),
input_source=('features1'))
# Create dataset which we will test our networks against.
shared_dataset = np.random.rand(20, 19).astype(theano.config.floatX)
# Make dataset for composite network.
dataset_composite = VectorSpacesDataset(
(shared_dataset[:, 0:5], shared_dataset[:, 5:15],
shared_dataset[:, 15:19]), (CompositeSpace(
[VectorSpace(5), VectorSpace(10),
VectorSpace(4)]), ('features0', 'features1', 'targets')))
# Make dataset for first single linear layer network.
dataset_first_part = VectorSpacesDataset(
(shared_dataset[:, 0:5], shared_dataset[:, 15:17]),
(CompositeSpace([VectorSpace(5), VectorSpace(2)]),
('features0', 'targets')))
# Make dataset for second single linear layer network.
dataset_second_part = VectorSpacesDataset(
(shared_dataset[:, 5:15], shared_dataset[:, 17:19]),
(CompositeSpace([VectorSpace(10), VectorSpace(2)]),
('features1', 'targets')))
# Initialize all MLPs to start from zero weights.
mlp_composite.layers[0].raw_layer.layers[0].set_weights(
mlp_composite.layers[0].raw_layer.layers[0].get_weights() * 0.0)
mlp_composite.layers[0].raw_layer.layers[1].set_weights(
mlp_composite.layers[0].raw_layer.layers[1].get_weights() * 0.0)
mlp_first_part.layers[0].set_weights(
mlp_first_part.layers[0].get_weights() * 0.0)
mlp_second_part.layers[0].set_weights(
mlp_second_part.layers[0].get_weights() * 0.0)
# Train all models with their respective datasets.
train_composite = Train(dataset_composite, mlp_composite,
SGD(0.0001, batch_size=20))
train_composite.algorithm.termination_criterion = EpochCounter(1)
train_composite.main_loop()
train_first_part = Train(dataset_first_part, mlp_first_part,
SGD(0.0001, batch_size=20))
train_first_part.algorithm.termination_criterion = EpochCounter(1)
train_first_part.main_loop()
train_second_part = Train(dataset_second_part, mlp_second_part,
SGD(0.0001, batch_size=20))
train_second_part.algorithm.termination_criterion = EpochCounter(1)
train_second_part.main_loop()
# Check that the composite feed-forward neural network has learned
# same parameters as each individual feed-forward neural network.
np.testing.assert_allclose(
mlp_composite.layers[0].raw_layer.layers[0].get_weights(),
mlp_first_part.layers[0].get_weights())
np.testing.assert_allclose(
mlp_composite.layers[0].raw_layer.layers[1].get_weights(),
mlp_second_part.layers[0].get_weights())
# Check that we get same output given the same input on a randomly
# generated dataset.
X_composite = mlp_composite.get_input_space().make_theano_batch()
X_first_part = mlp_first_part.get_input_space().make_theano_batch()
X_second_part = mlp_second_part.get_input_space().make_theano_batch()
fprop_composite = theano.function(X_composite,
mlp_composite.fprop(X_composite))
fprop_first_part = theano.function([X_first_part],
mlp_first_part.fprop(X_first_part))
fprop_second_part = theano.function([X_second_part],
mlp_second_part.fprop(X_second_part))
X_data = np.random.random(size=(10, 15)).astype(theano.config.floatX)
y_data = np.random.randint(low=0, high=10, size=(10, 4))
np.testing.assert_allclose(
fprop_composite(X_data[:, 0:5], X_data[:, 5:15])[:, 0:2],
fprop_first_part(X_data[:, 0:5]))
np.testing.assert_allclose(
fprop_composite(X_data[:, 0:5], X_data[:, 5:15])[:, 2:4],
fprop_second_part(X_data[:, 5:15]))
# Finally check that calling the internal FlattenerLayer behaves
# as we would expect. First, retrieve the FlattenerLayer.
fl = mlp_composite.layers[0]
# Check that it agrees on the input space.
assert mlp_composite.get_input_space() == fl.get_input_space()
# Check that it agrees on the parameters.
for i in range(0, 4):
np.testing.assert_allclose(fl.get_params()[i].eval(),
mlp_composite.get_params()[i].eval())
CompositeLayer(
layer_name='composite',
layers=[
MLP(layers=[
SpaceConverter(
'spconveter',
Conv2DSpace(shape=[32, 32],
num_channels=3,
axes=['c', 0, 1, 'b'])),
PretrainedMLP(
layer_name='gcn_column',
layer_content=serial.load(
'pkl/best/singlecolumn_complex_GCN_paper_best.pkl')
),
]),
MLP(layers=[
SpaceConverter(
'spconveter',
Conv2DSpace(shape=[32, 32],
num_channels=3,
axes=['c', 0, 1, 'b'])),
# PretrainedColumn(layer_name='toronto_column',
# layer_content=serial.load('pkl/best/singlecolumn_complex_TORONTO_paper_best.pkl')),
PretrainedMLP(
layer_name='zca_column',
layer_content=serial.load(
'pkl/best/singlecolumn_complex_ZCA_paper_best.pkl')
),
]),
],
inputs_to_layers={
0: [0],
1: [1]
}
# ),
),