def test_multiple_inputs(): """ Create a VectorSpacesDataset with two inputs (features0 and features1) and train an MLP which takes both inputs for 1 epoch. """ mlp = MLP(layers=[ FlattenerLayer( CompositeLayer('composite', [Linear(10, 'h0', 0.1), Linear(10, 'h1', 0.1)], { 0: [1], 1: [0] })), Softmax(5, 'softmax', 0.1) ], input_space=CompositeSpace([VectorSpace(15), VectorSpace(20)]), input_source=('features0', 'features1')) dataset = VectorSpacesDataset( (np.random.rand(20, 20).astype(theano.config.floatX), np.random.rand(20, 15).astype(theano.config.floatX), np.random.rand(20, 5).astype(theano.config.floatX)), (CompositeSpace( [VectorSpace(20), VectorSpace(15), VectorSpace(5)]), ('features1', 'features0', 'targets'))) train = Train(dataset, mlp, SGD(0.1, batch_size=5)) train.algorithm.termination_criterion = EpochCounter(1) train.main_loop()
def test_get_layer_monitor_channels(): """ Create a MLP with multiple layer types and get layer monitoring channels for MLP. """ mlp = MLP(layers=[ FlattenerLayer( CompositeLayer('composite', [Linear(10, 'h0', 0.1), Linear(10, 'h1', 0.1)], { 0: [1], 1: [0] })), Softmax(5, 'softmax', 0.1) ], input_space=CompositeSpace([VectorSpace(15), VectorSpace(20)]), input_source=('features0', 'features1')) dataset = VectorSpacesDataset( (np.random.rand(20, 20).astype(theano.config.floatX), np.random.rand(20, 15).astype(theano.config.floatX), np.random.rand(20, 5).astype(theano.config.floatX)), (CompositeSpace( [VectorSpace(20), VectorSpace(15), VectorSpace(5)]), ('features1', 'features0', 'targets'))) state_below = mlp.get_input_space().make_theano_batch() targets = mlp.get_target_space().make_theano_batch() mlp.get_layer_monitoring_channels(state_below=state_below, state=None, targets=targets)
def test_input_and_target_source(): """ Create a MLP and test input_source and target_source for default and non-default options. """ mlp = MLP( layers=[CompositeLayer( 'composite', [Linear(10, 'h0', 0.1), Linear(10, 'h1', 0.1)], { 0: [1], 1: [0] } ) ], input_space=CompositeSpace([VectorSpace(15), VectorSpace(20)]), input_source=('features0', 'features1'), target_source=('targets0', 'targets1') ) np.testing.assert_equal(mlp.get_input_source(), ('features0', 'features1')) np.testing.assert_equal(mlp.get_target_source(), ('targets0', 'targets1')) mlp = MLP( layers=[Linear(10, 'h0', 0.1)], input_space=VectorSpace(15) ) np.testing.assert_equal(mlp.get_input_source(), 'features') np.testing.assert_equal(mlp.get_target_source(), 'targets')
def _get_default_output_layer(self): return CompositeLayer(layer_name='conditional', layers=[ Linear(dim=self.ndim, layer_name='mu', irange=0.01), Linear(dim=self.ndim, layer_name='log_sigma', irange=0.01) ])
def _get_default_output_layer(self): return CompositeLayer(layer_name='composite', layers=[ Linear(layer_name='1', dim=self.ndim, irange=0.01), Linear(layer_name='2', dim=self.ndim, irange=0.01) ])
def test_nested_mlp(): """ Constructs a nested MLP and tries to fprop through it """ inner_mlp = MLP(layers=[Linear(10, 'h0', 0.1), Linear(10, 'h1', 0.1)], layer_name='inner_mlp') outer_mlp = MLP(layers=[CompositeLayer(layer_name='composite', layers=[inner_mlp, Linear(10, 'h2', 0.1)])], nvis=10) X = outer_mlp.get_input_space().make_theano_batch() f = theano.function([X], outer_mlp.fprop(X)) f(np.random.rand(5, 10).astype(theano.config.floatX))
def create_input_layer(self): size = self.extender_dim * (self.ws * 2 + 1) embed = Linear(layer_name='lin_embed', dim=size, istdev=1. / sqrt(size)) return CompositeLayer( layer_name='input', layers=[ ProjectionLayer(layer_name='ext_words', dim=self.edim, irange=.1), ProjectionLayer(layer_name='ext_feats', dim=self.fedim, irange=.1), embed, ], inputs_to_layers={0: [0], 1: [1], 2: [2]} )
def create_input_layer(self): return CompositeLayer(layer_name='input', layers=[ ProjectionLayer(layer_name='words', dim=self.edim, irange=.1), ProjectionLayer(layer_name='feats', dim=self.fedim, irange=.1), ], inputs_to_layers={ 0: [0], 1: [1] })
def test_flattener_layer_state_separation_for_conv(): """ Creates a CompositeLayer wrapping two Conv layers and ensures that state gets correctly picked apart. """ conv1 = ConvElemwise(8, [2, 2], 'sf1', SigmoidConvNonlinearity(), .1) conv2 = ConvElemwise(8, [2, 2], 'sf2', SigmoidConvNonlinearity(), .1) mlp = MLP(layers=[FlattenerLayer(CompositeLayer('comp', [conv1, conv2]))], input_space=Conv2DSpace(shape=[5, 5], num_channels=2)) topo_view = np.random.rand(10, 5, 5, 2).astype(theano.config.floatX) y = np.random.rand(10, 256).astype(theano.config.floatX) dataset = DenseDesignMatrix(topo_view=topo_view, y=y) train = Train(dataset, mlp, SGD(0.1, batch_size=5, monitoring_dataset=dataset)) train.algorithm.termination_criterion = EpochCounter(1) train.main_loop()
def test_flattener_layer_state_separation_for_softmax(): """ Creates a CompositeLayer wrapping two Softmax layers and ensures that state gets correctly picked apart. """ soft1 = Softmax(5, 'sf1', .1) soft2 = Softmax(5, 'sf2', .1) mlp = MLP(layers=[FlattenerLayer(CompositeLayer('comp', [soft1, soft2]))], nvis=2) X = np.random.rand(20, 2).astype(theano.config.floatX) y = np.random.rand(20, 10).astype(theano.config.floatX) dataset = DenseDesignMatrix(X=X, y=y) train = Train(dataset, mlp, SGD(0.1, batch_size=5, monitoring_dataset=dataset)) train.algorithm.termination_criterion = EpochCounter(1) train.main_loop()
def test_composite_layer(): """ Test the routing functionality of the CompositeLayer """ # Without routing composite_layer = CompositeLayer('composite_layer', [Linear(2, 'h0', irange=0), Linear(2, 'h1', irange=0), Linear(2, 'h2', irange=0)]) mlp = MLP(nvis=2, layers=[composite_layer]) for i in range(3): composite_layer.layers[i].set_weights( np.eye(2, dtype=theano.config.floatX) ) composite_layer.layers[i].set_biases( np.zeros(2, dtype=theano.config.floatX) ) X = tensor.matrix() y = mlp.fprop(X) funs = [theano.function([X], y_elem) for y_elem in y] x_numeric = np.random.rand(2, 2).astype('float32') y_numeric = [f(x_numeric) for f in funs] assert np.all(x_numeric == y_numeric) # With routing for inputs_to_layers in [{0: [1], 1: [2], 2: [0]}, {0: [1], 1: [0, 2], 2: []}, {0: [], 1: []}]: composite_layer = CompositeLayer('composite_layer', [Linear(2, 'h0', irange=0), Linear(2, 'h1', irange=0), Linear(2, 'h2', irange=0)], inputs_to_layers) input_space = CompositeSpace([VectorSpace(dim=2), VectorSpace(dim=2), VectorSpace(dim=2)]) mlp = MLP(input_space=input_space, layers=[composite_layer]) for i in range(3): composite_layer.layers[i].set_weights( np.eye(2, dtype=theano.config.floatX) ) composite_layer.layers[i].set_biases( np.zeros(2, dtype=theano.config.floatX) ) X = [tensor.matrix() for _ in range(3)] y = mlp.fprop(X) funs = [theano.function(X, y_elem, on_unused_input='ignore') for y_elem in y] x_numeric = [np.random.rand(2, 2).astype(theano.config.floatX) for _ in range(3)] y_numeric = [f(*x_numeric) for f in funs] assert all([all([np.all(x_numeric[i] == y_numeric[j]) for j in inputs_to_layers[i]]) for i in inputs_to_layers]) # Get the weight decay expressions from a composite layer composite_layer = CompositeLayer('composite_layer', [Linear(2, 'h0', irange=0.1), Linear(2, 'h1', irange=0.1)]) input_space = VectorSpace(dim=10) mlp = MLP(input_space=input_space, layers=[composite_layer]) for attr, coeff in product(['get_weight_decay', 'get_l1_weight_decay'], [[0.7, 0.3], 0.5]): f = theano.function([], getattr(composite_layer, attr)(coeff)) if is_iterable(coeff): g = theano.function( [], tensor.sum([getattr(layer, attr)(c) for c, layer in zip(coeff, composite_layer.layers)]) ) assert np.allclose(f(), g()) else: g = theano.function( [], tensor.sum([getattr(layer, attr)(coeff) for layer in composite_layer.layers]) ) assert np.allclose(f(), g())
def test_flattener_layer(): # To test the FlattenerLayer we create a very simple feed-forward neural # network with two parallel linear layers. We then create two separate # feed-forward neural networks with single linear layers. In principle, # these two models should be identical if we start from the same # parameters. This makes it easy to test that the composite layer works # as expected. # Create network with composite layers. mlp_composite = MLP(layers=[ FlattenerLayer( CompositeLayer('composite', [Linear(2, 'h0', 0.1), Linear(2, 'h1', 0.1)], { 0: [0], 1: [1] })) ], input_space=CompositeSpace( [VectorSpace(5), VectorSpace(10)]), input_source=('features0', 'features1')) # Create network with single linear layer, corresponding to first # layer in the composite network. mlp_first_part = MLP(layers=[Linear(2, 'h0', 0.1)], input_space=VectorSpace(5), input_source=('features0')) # Create network with single linear layer, corresponding to second # layer in the composite network. mlp_second_part = MLP(layers=[Linear(2, 'h1', 0.1)], input_space=VectorSpace(10), input_source=('features1')) # Create dataset which we will test our networks against. shared_dataset = np.random.rand(20, 19).astype(theano.config.floatX) # Make dataset for composite network. dataset_composite = VectorSpacesDataset( (shared_dataset[:, 0:5], shared_dataset[:, 5:15], shared_dataset[:, 15:19]), (CompositeSpace( [VectorSpace(5), VectorSpace(10), VectorSpace(4)]), ('features0', 'features1', 'targets'))) # Make dataset for first single linear layer network. dataset_first_part = VectorSpacesDataset( (shared_dataset[:, 0:5], shared_dataset[:, 15:17]), (CompositeSpace([VectorSpace(5), VectorSpace(2)]), ('features0', 'targets'))) # Make dataset for second single linear layer network. dataset_second_part = VectorSpacesDataset( (shared_dataset[:, 5:15], shared_dataset[:, 17:19]), (CompositeSpace([VectorSpace(10), VectorSpace(2)]), ('features1', 'targets'))) # Initialize all MLPs to start from zero weights. mlp_composite.layers[0].raw_layer.layers[0].set_weights( mlp_composite.layers[0].raw_layer.layers[0].get_weights() * 0.0) mlp_composite.layers[0].raw_layer.layers[1].set_weights( mlp_composite.layers[0].raw_layer.layers[1].get_weights() * 0.0) mlp_first_part.layers[0].set_weights( mlp_first_part.layers[0].get_weights() * 0.0) mlp_second_part.layers[0].set_weights( mlp_second_part.layers[0].get_weights() * 0.0) # Train all models with their respective datasets. train_composite = Train(dataset_composite, mlp_composite, SGD(0.0001, batch_size=20)) train_composite.algorithm.termination_criterion = EpochCounter(1) train_composite.main_loop() train_first_part = Train(dataset_first_part, mlp_first_part, SGD(0.0001, batch_size=20)) train_first_part.algorithm.termination_criterion = EpochCounter(1) train_first_part.main_loop() train_second_part = Train(dataset_second_part, mlp_second_part, SGD(0.0001, batch_size=20)) train_second_part.algorithm.termination_criterion = EpochCounter(1) train_second_part.main_loop() # Check that the composite feed-forward neural network has learned # same parameters as each individual feed-forward neural network. np.testing.assert_allclose( mlp_composite.layers[0].raw_layer.layers[0].get_weights(), mlp_first_part.layers[0].get_weights()) np.testing.assert_allclose( mlp_composite.layers[0].raw_layer.layers[1].get_weights(), mlp_second_part.layers[0].get_weights()) # Check that we get same output given the same input on a randomly # generated dataset. X_composite = mlp_composite.get_input_space().make_theano_batch() X_first_part = mlp_first_part.get_input_space().make_theano_batch() X_second_part = mlp_second_part.get_input_space().make_theano_batch() fprop_composite = theano.function(X_composite, mlp_composite.fprop(X_composite)) fprop_first_part = theano.function([X_first_part], mlp_first_part.fprop(X_first_part)) fprop_second_part = theano.function([X_second_part], mlp_second_part.fprop(X_second_part)) X_data = np.random.random(size=(10, 15)).astype(theano.config.floatX) y_data = np.random.randint(low=0, high=10, size=(10, 4)) np.testing.assert_allclose( fprop_composite(X_data[:, 0:5], X_data[:, 5:15])[:, 0:2], fprop_first_part(X_data[:, 0:5])) np.testing.assert_allclose( fprop_composite(X_data[:, 0:5], X_data[:, 5:15])[:, 2:4], fprop_second_part(X_data[:, 5:15])) # Finally check that calling the internal FlattenerLayer behaves # as we would expect. First, retrieve the FlattenerLayer. fl = mlp_composite.layers[0] # Check that it agrees on the input space. assert mlp_composite.get_input_space() == fl.get_input_space() # Check that it agrees on the parameters. for i in range(0, 4): np.testing.assert_allclose(fl.get_params()[i].eval(), mlp_composite.get_params()[i].eval())
CompositeLayer( layer_name='composite', layers=[ MLP(layers=[ SpaceConverter( 'spconveter', Conv2DSpace(shape=[32, 32], num_channels=3, axes=['c', 0, 1, 'b'])), PretrainedMLP( layer_name='gcn_column', layer_content=serial.load( 'pkl/best/singlecolumn_complex_GCN_paper_best.pkl') ), ]), MLP(layers=[ SpaceConverter( 'spconveter', Conv2DSpace(shape=[32, 32], num_channels=3, axes=['c', 0, 1, 'b'])), # PretrainedColumn(layer_name='toronto_column', # layer_content=serial.load('pkl/best/singlecolumn_complex_TORONTO_paper_best.pkl')), PretrainedMLP( layer_name='zca_column', layer_content=serial.load( 'pkl/best/singlecolumn_complex_ZCA_paper_best.pkl') ), ]), ], inputs_to_layers={ 0: [0], 1: [1] } # ), ),