def test_multiple_inputs(): """ Create a VectorSpacesDataset with two inputs (features0 and features1) and train an MLP which takes both inputs for 1 epoch. """ mlp = MLP(layers=[ FlattenerLayer( CompositeLayer('composite', [Linear(10, 'h0', 0.1), Linear(10, 'h1', 0.1)], { 0: [1], 1: [0] })), Softmax(5, 'softmax', 0.1) ], input_space=CompositeSpace([VectorSpace(15), VectorSpace(20)]), input_source=('features0', 'features1')) dataset = VectorSpacesDataset( (np.random.rand(20, 20).astype(theano.config.floatX), np.random.rand(20, 15).astype(theano.config.floatX), np.random.rand(20, 5).astype(theano.config.floatX)), (CompositeSpace( [VectorSpace(20), VectorSpace(15), VectorSpace(5)]), ('features1', 'features0', 'targets'))) train = Train(dataset, mlp, SGD(0.1, batch_size=5)) train.algorithm.termination_criterion = EpochCounter(1) train.main_loop()
def test_get_layer_monitor_channels(): """ Create a MLP with multiple layer types and get layer monitoring channels for MLP. """ mlp = MLP(layers=[ FlattenerLayer( CompositeLayer('composite', [Linear(10, 'h0', 0.1), Linear(10, 'h1', 0.1)], { 0: [1], 1: [0] })), Softmax(5, 'softmax', 0.1) ], input_space=CompositeSpace([VectorSpace(15), VectorSpace(20)]), input_source=('features0', 'features1')) dataset = VectorSpacesDataset( (np.random.rand(20, 20).astype(theano.config.floatX), np.random.rand(20, 15).astype(theano.config.floatX), np.random.rand(20, 5).astype(theano.config.floatX)), (CompositeSpace( [VectorSpace(20), VectorSpace(15), VectorSpace(5)]), ('features1', 'features0', 'targets'))) state_below = mlp.get_input_space().make_theano_batch() targets = mlp.get_target_space().make_theano_batch() mlp.get_layer_monitoring_channels(state_below=state_below, state=None, targets=targets)
def test_input_and_target_source(): """ Create a MLP and test input_source and target_source for default and non-default options. """ mlp = MLP( layers=[CompositeLayer( 'composite', [Linear(10, 'h0', 0.1), Linear(10, 'h1', 0.1)], { 0: [1], 1: [0] } ) ], input_space=CompositeSpace([VectorSpace(15), VectorSpace(20)]), input_source=('features0', 'features1'), target_source=('targets0', 'targets1') ) np.testing.assert_equal(mlp.get_input_source(), ('features0', 'features1')) np.testing.assert_equal(mlp.get_target_source(), ('targets0', 'targets1')) mlp = MLP( layers=[Linear(10, 'h0', 0.1)], input_space=VectorSpace(15) ) np.testing.assert_equal(mlp.get_input_source(), 'features') np.testing.assert_equal(mlp.get_target_source(), 'targets')
def test_masked_fprop(): # Construct a dirt-simple linear network with identity weights. mlp = MLP(nvis=2, layers=[Linear(2, 'h0', irange=0), Linear(2, 'h1', irange=0)]) mlp.layers[0].set_weights(np.eye(2, dtype=mlp.get_weights().dtype)) mlp.layers[1].set_weights(np.eye(2, dtype=mlp.get_weights().dtype)) mlp.layers[0].set_biases(np.arange(1, 3, dtype=mlp.get_weights().dtype)) mlp.layers[1].set_biases(np.arange(3, 5, dtype=mlp.get_weights().dtype)) # Verify that get_total_input_dimension works. np.testing.assert_equal(mlp.get_total_input_dimension(['h0', 'h1']), 4) inp = theano.tensor.matrix() # Accumulate the sum of output of all masked networks. l = [] for mask in xrange(16): l.append(mlp.masked_fprop(inp, mask)) outsum = reduce(lambda x, y: x + y, l) f = theano.function([inp], outsum, allow_input_downcast=True) np.testing.assert_equal(f([[5, 3]]), [[144., 144.]]) np.testing.assert_equal(f([[2, 7]]), [[96., 208.]]) np.testing.assert_raises(ValueError, mlp.masked_fprop, inp, 22) np.testing.assert_raises(ValueError, mlp.masked_fprop, inp, 2, ['h3']) np.testing.assert_raises(ValueError, mlp.masked_fprop, inp, 2, None, 2., {'h3': 4})
def test_exhaustive_dropout_average(): # This is only a smoke test: verifies that it compiles and runs, # not any particular value. inp = theano.tensor.matrix() mlp = MLP(nvis=2, layers=[ Linear(2, 'h0', irange=0.8), Linear(2, 'h1', irange=0.8), Softmax(3, 'out', irange=0.8) ]) out = exhaustive_dropout_average(mlp, inp) f = theano.function([inp], out, allow_input_downcast=True) f([[2.3, 4.9]]) out = exhaustive_dropout_average(mlp, inp, input_scales={'h0': 3}) f = theano.function([inp], out, allow_input_downcast=True) f([[2.3, 4.9]]) out = exhaustive_dropout_average(mlp, inp, masked_input_layers=['h1']) f = theano.function([inp], out, allow_input_downcast=True) f([[2.3, 4.9]]) np.testing.assert_raises(ValueError, exhaustive_dropout_average, mlp, inp, ['h5']) np.testing.assert_raises(ValueError, exhaustive_dropout_average, mlp, inp, ['h0'], 2., {'h5': 3.})
def _get_default_output_layer(self): return CompositeLayer(layer_name='composite', layers=[ Linear(layer_name='1', dim=self.ndim, irange=0.01), Linear(layer_name='2', dim=self.ndim, irange=0.01) ])
def test_sampled_dropout_average(): # This is only a smoke test: verifies that it compiles and runs, # not any particular value. inp = theano.tensor.matrix() mlp = MLP(nvis=2, layers=[Linear(2, 'h0', irange=0.8), Linear(2, 'h1', irange=0.8), Softmax(3, 'out', irange=0.8)]) out = sampled_dropout_average(mlp, inp, 5) f = theano.function([inp], out, allow_input_downcast=True) f([[2.3, 4.9]])
def _get_default_output_layer(self): return CompositeLayer(layer_name='conditional', layers=[ Linear(dim=self.ndim, layer_name='mu', irange=0.01), Linear(dim=self.ndim, layer_name='log_sigma', irange=0.01) ])
def test_str(): """ Make sure the __str__ method returns a string """ mlp = MLP(nvis=2, layers=[Linear(2, 'h0', irange=0), Linear(2, 'h1', irange=0)]) s = str(mlp) assert isinstance(s, basestring)
def test_conditional_rejects_invalid_output_layer(): """ Conditional rejects invalid user-defined output layer """ mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01), Linear(layer_name='mu', dim=5, irange=0.01)]) conditional = DummyConditional(mlp=mlp, name='conditional', output_layer_required=False) vae = DummyVAE() conditional.set_vae(vae) conditional.initialize_parameters(input_space=VectorSpace(dim=5), ndim=5)
def test_exhaustive_dropout_average(): # This is only a smoke test: verifies that it compiles and runs, # not any particular value. inp = theano.tensor.matrix() mlp = MLP(nvis=2, layers=[ Linear(2, 'h0', irange=0.8), Linear(2, 'h1', irange=0.8), Softmax(3, 'out', irange=0.8) ]) out = exhaustive_dropout_average(mlp, inp) f = theano.function([inp], out) f([[2.3, 4.9]])
def test_nested_mlp(): """ Constructs a nested MLP and tries to fprop through it """ inner_mlp = MLP(layers=[Linear(10, 'h0', 0.1), Linear(10, 'h1', 0.1)], layer_name='inner_mlp') outer_mlp = MLP(layers=[CompositeLayer(layer_name='composite', layers=[inner_mlp, Linear(10, 'h2', 0.1)])], nvis=10) X = outer_mlp.get_input_space().make_theano_batch() f = theano.function([X], outer_mlp.fprop(X)) f(np.random.rand(5, 10).astype(theano.config.floatX))
def test_vae_automatically_finds_kl_integrator(): """ VAE automatically finds the right KLIntegrator """ encoding_model = MLP(layers=[Linear(layer_name='h', dim=10, irange=0.01)]) decoding_model = MLP(layers=[Linear(layer_name='h', dim=10, irange=0.01)]) prior = DiagonalGaussianPrior() conditional = BernoulliVector(mlp=decoding_model, name='conditional') posterior = DiagonalGaussian(mlp=encoding_model, name='posterior') vae = VAE(nvis=10, prior=prior, conditional=conditional, posterior=posterior, nhid=5) assert (vae.kl_integrator is not None and isinstance(vae.kl_integrator, DiagonalGaussianPriorPosteriorKL))
def get_layer_linear(self, layer_id, layer_name): row = self.db.executeSQL( """ SELECT dim,irange,istdev,sparse_init,sparse_stdev,include_prob, init_bias,W_lr_scale,b_lr_scale,max_row_norm, max_col_norm,softmax_columns FROM hps3.layer_linear WHERE layer_id = %s """, (layer_id, ), self.db.FETCH_ONE) if not row or row is None: raise HPSData("No linear layer for layer_id="\ +str(layer_id)) (dim, irange, istdev, sparse_init, sparse_stdev, include_prob, init_bias, W_lr_scale, b_lr_scale, max_row_norm, max_col_norm, softmax_columns) = row return Linear(dim=dim, irange=irange, istdev=istdev, sparse_init=sparse_init, sparse_stdev=sparse_stdev, include_prob=include_prob, init_bias=init_bias, W_lr_scale=W_lr_scale, b_lr_scale=b_lr_scale, max_row_norm=max_row_norm, max_col_norm=max_col_norm, layer_name=layer_name, softmax_columns=softmax_columns)
def test_convolutional_compatible(): """ VAE allows convolutional encoding networks """ encoding_model = MLP( layers=[ SpaceConverter( layer_name='conv2d_converter', output_space=Conv2DSpace(shape=[4, 4], num_channels=1) ), ConvRectifiedLinear( layer_name='h', output_channels=2, kernel_shape=[2, 2], kernel_stride=[1, 1], pool_shape=[1, 1], pool_stride=[1, 1], pool_type='max', irange=0.01) ] ) decoding_model = MLP(layers=[Linear(layer_name='h', dim=16, irange=0.01)]) prior = DiagonalGaussianPrior() conditional = BernoulliVector(mlp=decoding_model, name='conditional') posterior = DiagonalGaussian(mlp=encoding_model, name='posterior') vae = VAE(nvis=16, prior=prior, conditional=conditional, posterior=posterior, nhid=16) X = T.matrix('X') lower_bound = vae.log_likelihood_lower_bound(X, num_samples=10) f = theano.function(inputs=[X], outputs=lower_bound) rng = make_np_rng(default_seed=11223) f(as_floatX(rng.uniform(size=(10, 16))))
def test_multiple_samples_allowed(): """ VAE allows multiple samples per data point """ encoding_model = MLP(layers=[Linear(layer_name='h', dim=10, irange=0.01)]) decoding_model = MLP(layers=[Linear(layer_name='h', dim=10, irange=0.01)]) prior = DiagonalGaussianPrior() conditional = BernoulliVector(mlp=decoding_model, name='conditional') posterior = DiagonalGaussian(mlp=encoding_model, name='posterior') vae = VAE(nvis=10, prior=prior, conditional=conditional, posterior=posterior, nhid=5) X = T.matrix('X') lower_bound = vae.log_likelihood_lower_bound(X, num_samples=10) f = theano.function(inputs=[X], outputs=lower_bound) rng = make_np_rng(default_seed=11223) f(as_floatX(rng.uniform(size=(10, 10))))
def test_gradient_clipping(self): """ Create a known gradient and check whether it is being clipped correctly """ mlp = MLP(layers=[Linear(dim=1, irange=0, layer_name='linear')], nvis=1) W, b = mlp.layers[0].get_params() W.set_value([[10]]) X = mlp.get_input_space().make_theano_batch() y = mlp.get_output_space().make_theano_batch() cost = Default() gradients, _ = cost.get_gradients(mlp, (X, y)) clipped_cost = GradientClipping(20, Default()) clipped_gradients, _ = clipped_cost.get_gradients(mlp, (X, y)) # The MLP defines f(x) = (x W)^2, with df/dW = 2 W x^2 f = function([X, y], [gradients[W].sum(), clipped_gradients[W].sum()], allow_input_downcast=True) # df/dW = df/db = 20 for W = 10, x = 1, so the norm is 20 * sqrt(2) # and the gradients should be clipped to 20 / sqrt(2) np.testing.assert_allclose(f([[1]], [[0]]), [20, 20 / np.sqrt(2)])
def test_conditional_initialize_parameters(): """ Conditional.initialize_parameters does the following: * Set its input_space and ndim attributes * Calls its MLP's set_mlp method * Sets its MLP's input_space * Validates its MLP * Sets its params and param names """ mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01, max_col_norm=0.01)]) conditional = DummyConditional(mlp=mlp, name='conditional') vae = DummyVAE() conditional.set_vae(vae) input_space = VectorSpace(dim=5) conditional.initialize_parameters(input_space=input_space, ndim=5) testing.assert_same_object(input_space, conditional.input_space) testing.assert_equal(conditional.ndim, 5) testing.assert_same_object(mlp.get_mlp(), conditional) testing.assert_same_object(mlp.input_space, input_space) mlp_params = mlp.get_params() conditional_params = conditional.get_params() assert all([mp in conditional_params for mp in mlp_params]) assert all([cp in mlp_params for cp in conditional_params])
def create_output_layer(self): sc = 1. / self.n_classes output = Linear(layer_name='tagger_out', istdev=1. / sqrt(self.n_classes), dim=self.n_classes, W_lr_scale=sc, b_lr_scale=sc) return output
def test_conditional_get_vae(): """ Conditional.get_vae returns its VAE """ mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01)]) conditional = DummyConditional(mlp=mlp, name='conditional') vae = DummyVAE() conditional.set_vae(vae) testing.assert_same_object(conditional.get_vae(), vae)
def test_conditional_raises_exception_if_called_twice(): """ Conditional.set_vae raises an exception if it has already been called """ mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01)]) conditional = DummyConditional(mlp=mlp, name='conditional') vae = DummyVAE() conditional.set_vae(vae) conditional.set_vae(vae)
def test_conditional_returns_mlp_weights(): """ Conditional.get_weights calls its MLP's get_weights method """ mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01)]) conditional = DummyConditional(mlp=mlp, name='conditional') vae = DummyVAE() conditional.set_vae(vae) conditional.initialize_parameters(input_space=VectorSpace(dim=5), ndim=5) numpy.testing.assert_equal(conditional.get_weights(), mlp.get_weights())
def get_Linear_Layer(self, structure, i=0): n_input, n_output = structure config = { 'dim': n_output, 'layer_name': ("l%d" % i), 'irange': .5, 'use_abs_loss': False, 'use_bias': False, } return Linear(**config)
def test_summarize_model(): """ Asks the summarize_model.py script to inspect a pickled model and check that it completes succesfully """ skip_if_no_matplotlib() with open('model.pkl', 'wb') as f: cPickle.dump(MLP(layers=[Linear(dim=5, layer_name='h0', irange=0.1)], nvis=10), f, protocol=cPickle.HIGHEST_PROTOCOL) summarize('model.pkl') os.remove('model.pkl')
def test_diagonal_gaussian_default_output_layer(): """ DiagonalGaussian's default output layer is compatible with its required output space """ mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01, max_col_norm=0.01)]) conditional = DiagonalGaussian(mlp=mlp, name='conditional') vae = DummyVAE() conditional.set_vae(vae) input_space = VectorSpace(dim=5) conditional.initialize_parameters(input_space=input_space, ndim=5)
def test_diagonal_gaussian_sample_from_epsilon(): """ DiagonalGaussian.sample_from_epsilon doesn't crash """ mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01, max_col_norm=0.01)]) conditional = DiagonalGaussian(mlp=mlp, name='conditional') vae = DummyVAE() conditional.set_vae(vae) input_space = VectorSpace(dim=5) conditional.initialize_parameters(input_space=input_space, ndim=5) conditional.sample_from_epsilon((2, 10, 5))
def test_conditional_set_vae(): """ Conditional.set_vae adds a reference to the vae and adopts the vae's rng and batch_size attributes """ mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01)]) conditional = DummyConditional(mlp=mlp, name='conditional') vae = DummyVAE() conditional.set_vae(vae) testing.assert_same_object(conditional.vae, vae) testing.assert_same_object(conditional.rng, vae.rng) testing.assert_equal(conditional.batch_size, vae.batch_size)
def test_bernoulli_vector_sample_from_conditional(): """ BernoulliVector.sample_from_conditional works when num_samples is provided """ mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01, max_col_norm=0.01)]) conditional = BernoulliVector(mlp=mlp, name='conditional') vae = DummyVAE() conditional.set_vae(vae) input_space = VectorSpace(dim=5) conditional.initialize_parameters(input_space=input_space, ndim=5) mu = T.matrix('mu') conditional.sample_from_conditional([mu], num_samples=2)
def test_bernoulli_vector_conditional_expectation(): """ BernoulliVector.conditional_expectation doesn't crash """ mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01, max_col_norm=0.01)]) conditional = BernoulliVector(mlp=mlp, name='conditional') vae = DummyVAE() conditional.set_vae(vae) input_space = VectorSpace(dim=5) conditional.initialize_parameters(input_space=input_space, ndim=5) mu = T.matrix('mu') conditional.conditional_expectation([mu])
def test_conditional_modify_updates(): """ Conditional.modify_updates calls its MLP's modify_updates method """ mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01, max_col_norm=0.01)]) conditional = DummyConditional(mlp=mlp, name='conditional') vae = DummyVAE() conditional.set_vae(vae) conditional.initialize_parameters(input_space=VectorSpace(dim=5), ndim=5) updates = OrderedDict(zip(mlp.get_params(), mlp.get_params())) testing.assert_equal(conditional.modify_updates(updates), mlp.modify_updates(updates))