def test_multiple_inputs():
"""
Create a VectorSpacesDataset with two inputs (features0 and features1)
and train an MLP which takes both inputs for 1 epoch.
"""
mlp = MLP(layers=[
FlattenerLayer(
CompositeLayer('composite',
[Linear(10, 'h0', 0.1),
Linear(10, 'h1', 0.1)], {
0: [1],
1: [0]
})),
Softmax(5, 'softmax', 0.1)
],
input_space=CompositeSpace([VectorSpace(15),
VectorSpace(20)]),
input_source=('features0', 'features1'))
dataset = VectorSpacesDataset(
(np.random.rand(20, 20).astype(theano.config.floatX),
np.random.rand(20, 15).astype(theano.config.floatX),
np.random.rand(20, 5).astype(theano.config.floatX)),
(CompositeSpace(
[VectorSpace(20), VectorSpace(15),
VectorSpace(5)]), ('features1', 'features0', 'targets')))
train = Train(dataset, mlp, SGD(0.1, batch_size=5))
train.algorithm.termination_criterion = EpochCounter(1)
train.main_loop()
def test_get_layer_monitor_channels():
"""
Create a MLP with multiple layer types
and get layer monitoring channels for MLP.
"""
mlp = MLP(layers=[
FlattenerLayer(
CompositeLayer('composite',
[Linear(10, 'h0', 0.1),
Linear(10, 'h1', 0.1)], {
0: [1],
1: [0]
})),
Softmax(5, 'softmax', 0.1)
],
input_space=CompositeSpace([VectorSpace(15),
VectorSpace(20)]),
input_source=('features0', 'features1'))
dataset = VectorSpacesDataset(
(np.random.rand(20, 20).astype(theano.config.floatX),
np.random.rand(20, 15).astype(theano.config.floatX),
np.random.rand(20, 5).astype(theano.config.floatX)),
(CompositeSpace(
[VectorSpace(20), VectorSpace(15),
VectorSpace(5)]), ('features1', 'features0', 'targets')))
state_below = mlp.get_input_space().make_theano_batch()
targets = mlp.get_target_space().make_theano_batch()
mlp.get_layer_monitoring_channels(state_below=state_below,
state=None,
targets=targets)
def test_input_and_target_source():
"""
Create a MLP and test input_source and target_source
for default and non-default options.
"""
mlp = MLP(
layers=[CompositeLayer(
'composite',
[Linear(10, 'h0', 0.1),
Linear(10, 'h1', 0.1)],
{
0: [1],
1: [0]
}
)
],
input_space=CompositeSpace([VectorSpace(15), VectorSpace(20)]),
input_source=('features0', 'features1'),
target_source=('targets0', 'targets1')
)
np.testing.assert_equal(mlp.get_input_source(), ('features0', 'features1'))
np.testing.assert_equal(mlp.get_target_source(), ('targets0', 'targets1'))
mlp = MLP(
layers=[Linear(10, 'h0', 0.1)],
input_space=VectorSpace(15)
)
np.testing.assert_equal(mlp.get_input_source(), 'features')
np.testing.assert_equal(mlp.get_target_source(), 'targets')
def test_masked_fprop():
# Construct a dirt-simple linear network with identity weights.
mlp = MLP(nvis=2,
layers=[Linear(2, 'h0', irange=0),
Linear(2, 'h1', irange=0)])
mlp.layers[0].set_weights(np.eye(2, dtype=mlp.get_weights().dtype))
mlp.layers[1].set_weights(np.eye(2, dtype=mlp.get_weights().dtype))
mlp.layers[0].set_biases(np.arange(1, 3, dtype=mlp.get_weights().dtype))
mlp.layers[1].set_biases(np.arange(3, 5, dtype=mlp.get_weights().dtype))
# Verify that get_total_input_dimension works.
np.testing.assert_equal(mlp.get_total_input_dimension(['h0', 'h1']), 4)
inp = theano.tensor.matrix()
# Accumulate the sum of output of all masked networks.
l = []
for mask in xrange(16):
l.append(mlp.masked_fprop(inp, mask))
outsum = reduce(lambda x, y: x + y, l)
f = theano.function([inp], outsum, allow_input_downcast=True)
np.testing.assert_equal(f([[5, 3]]), [[144., 144.]])
np.testing.assert_equal(f([[2, 7]]), [[96., 208.]])
np.testing.assert_raises(ValueError, mlp.masked_fprop, inp, 22)
np.testing.assert_raises(ValueError, mlp.masked_fprop, inp, 2, ['h3'])
np.testing.assert_raises(ValueError, mlp.masked_fprop, inp, 2, None, 2.,
{'h3': 4})
def test_exhaustive_dropout_average():
# This is only a smoke test: verifies that it compiles and runs,
# not any particular value.
inp = theano.tensor.matrix()
mlp = MLP(nvis=2,
layers=[
Linear(2, 'h0', irange=0.8),
Linear(2, 'h1', irange=0.8),
Softmax(3, 'out', irange=0.8)
])
out = exhaustive_dropout_average(mlp, inp)
f = theano.function([inp], out, allow_input_downcast=True)
f([[2.3, 4.9]])
out = exhaustive_dropout_average(mlp, inp, input_scales={'h0': 3})
f = theano.function([inp], out, allow_input_downcast=True)
f([[2.3, 4.9]])
out = exhaustive_dropout_average(mlp, inp, masked_input_layers=['h1'])
f = theano.function([inp], out, allow_input_downcast=True)
f([[2.3, 4.9]])
np.testing.assert_raises(ValueError, exhaustive_dropout_average, mlp, inp,
['h5'])
np.testing.assert_raises(ValueError, exhaustive_dropout_average, mlp, inp,
['h0'], 2., {'h5': 3.})
def _get_default_output_layer(self):
return CompositeLayer(layer_name='composite',
layers=[
Linear(layer_name='1',
dim=self.ndim,
irange=0.01),
Linear(layer_name='2',
dim=self.ndim,
irange=0.01)
])
def test_sampled_dropout_average():
# This is only a smoke test: verifies that it compiles and runs,
# not any particular value.
inp = theano.tensor.matrix()
mlp = MLP(nvis=2, layers=[Linear(2, 'h0', irange=0.8),
Linear(2, 'h1', irange=0.8),
Softmax(3, 'out', irange=0.8)])
out = sampled_dropout_average(mlp, inp, 5)
f = theano.function([inp], out, allow_input_downcast=True)
f([[2.3, 4.9]])
def _get_default_output_layer(self):
return CompositeLayer(layer_name='conditional',
layers=[
Linear(dim=self.ndim,
layer_name='mu',
irange=0.01),
Linear(dim=self.ndim,
layer_name='log_sigma',
irange=0.01)
])
def test_str():
"""
Make sure the __str__ method returns a string
"""
mlp = MLP(nvis=2, layers=[Linear(2, 'h0', irange=0),
Linear(2, 'h1', irange=0)])
s = str(mlp)
assert isinstance(s, basestring)
def test_conditional_rejects_invalid_output_layer():
"""
Conditional rejects invalid user-defined output layer
"""
mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01),
Linear(layer_name='mu', dim=5, irange=0.01)])
conditional = DummyConditional(mlp=mlp, name='conditional',
output_layer_required=False)
vae = DummyVAE()
conditional.set_vae(vae)
conditional.initialize_parameters(input_space=VectorSpace(dim=5), ndim=5)
def test_exhaustive_dropout_average():
# This is only a smoke test: verifies that it compiles and runs,
# not any particular value.
inp = theano.tensor.matrix()
mlp = MLP(nvis=2,
layers=[
Linear(2, 'h0', irange=0.8),
Linear(2, 'h1', irange=0.8),
Softmax(3, 'out', irange=0.8)
])
out = exhaustive_dropout_average(mlp, inp)
f = theano.function([inp], out)
f([[2.3, 4.9]])
def test_nested_mlp():
"""
Constructs a nested MLP and tries to fprop through it
"""
inner_mlp = MLP(layers=[Linear(10, 'h0', 0.1), Linear(10, 'h1', 0.1)],
layer_name='inner_mlp')
outer_mlp = MLP(layers=[CompositeLayer(layer_name='composite',
layers=[inner_mlp,
Linear(10, 'h2', 0.1)])],
nvis=10)
X = outer_mlp.get_input_space().make_theano_batch()
f = theano.function([X], outer_mlp.fprop(X))
f(np.random.rand(5, 10).astype(theano.config.floatX))
def test_vae_automatically_finds_kl_integrator():
"""
VAE automatically finds the right KLIntegrator
"""
encoding_model = MLP(layers=[Linear(layer_name='h', dim=10, irange=0.01)])
decoding_model = MLP(layers=[Linear(layer_name='h', dim=10, irange=0.01)])
prior = DiagonalGaussianPrior()
conditional = BernoulliVector(mlp=decoding_model, name='conditional')
posterior = DiagonalGaussian(mlp=encoding_model, name='posterior')
vae = VAE(nvis=10, prior=prior, conditional=conditional,
posterior=posterior, nhid=5)
assert (vae.kl_integrator is not None and
isinstance(vae.kl_integrator, DiagonalGaussianPriorPosteriorKL))
def get_layer_linear(self, layer_id, layer_name):
row = self.db.executeSQL(
"""
SELECT dim,irange,istdev,sparse_init,sparse_stdev,include_prob,
init_bias,W_lr_scale,b_lr_scale,max_row_norm,
max_col_norm,softmax_columns
FROM hps3.layer_linear
WHERE layer_id = %s
""", (layer_id, ), self.db.FETCH_ONE)
if not row or row is None:
raise HPSData("No linear layer for layer_id="\
+str(layer_id))
(dim, irange, istdev, sparse_init, sparse_stdev, include_prob,
init_bias, W_lr_scale, b_lr_scale, max_row_norm, max_col_norm,
softmax_columns) = row
return Linear(dim=dim,
irange=irange,
istdev=istdev,
sparse_init=sparse_init,
sparse_stdev=sparse_stdev,
include_prob=include_prob,
init_bias=init_bias,
W_lr_scale=W_lr_scale,
b_lr_scale=b_lr_scale,
max_row_norm=max_row_norm,
max_col_norm=max_col_norm,
layer_name=layer_name,
softmax_columns=softmax_columns)
def test_convolutional_compatible():
"""
VAE allows convolutional encoding networks
"""
encoding_model = MLP(
layers=[
SpaceConverter(
layer_name='conv2d_converter',
output_space=Conv2DSpace(shape=[4, 4], num_channels=1)
),
ConvRectifiedLinear(
layer_name='h',
output_channels=2,
kernel_shape=[2, 2],
kernel_stride=[1, 1],
pool_shape=[1, 1],
pool_stride=[1, 1],
pool_type='max',
irange=0.01)
]
)
decoding_model = MLP(layers=[Linear(layer_name='h', dim=16, irange=0.01)])
prior = DiagonalGaussianPrior()
conditional = BernoulliVector(mlp=decoding_model, name='conditional')
posterior = DiagonalGaussian(mlp=encoding_model, name='posterior')
vae = VAE(nvis=16, prior=prior, conditional=conditional,
posterior=posterior, nhid=16)
X = T.matrix('X')
lower_bound = vae.log_likelihood_lower_bound(X, num_samples=10)
f = theano.function(inputs=[X], outputs=lower_bound)
rng = make_np_rng(default_seed=11223)
f(as_floatX(rng.uniform(size=(10, 16))))
def test_multiple_samples_allowed():
"""
VAE allows multiple samples per data point
"""
encoding_model = MLP(layers=[Linear(layer_name='h', dim=10, irange=0.01)])
decoding_model = MLP(layers=[Linear(layer_name='h', dim=10, irange=0.01)])
prior = DiagonalGaussianPrior()
conditional = BernoulliVector(mlp=decoding_model, name='conditional')
posterior = DiagonalGaussian(mlp=encoding_model, name='posterior')
vae = VAE(nvis=10, prior=prior, conditional=conditional,
posterior=posterior, nhid=5)
X = T.matrix('X')
lower_bound = vae.log_likelihood_lower_bound(X, num_samples=10)
f = theano.function(inputs=[X], outputs=lower_bound)
rng = make_np_rng(default_seed=11223)
f(as_floatX(rng.uniform(size=(10, 10))))
def test_gradient_clipping(self):
"""
Create a known gradient and check whether it is being clipped
correctly
"""
mlp = MLP(layers=[Linear(dim=1, irange=0, layer_name='linear')],
nvis=1)
W, b = mlp.layers[0].get_params()
W.set_value([[10]])
X = mlp.get_input_space().make_theano_batch()
y = mlp.get_output_space().make_theano_batch()
cost = Default()
gradients, _ = cost.get_gradients(mlp, (X, y))
clipped_cost = GradientClipping(20, Default())
clipped_gradients, _ = clipped_cost.get_gradients(mlp, (X, y))
# The MLP defines f(x) = (x W)^2, with df/dW = 2 W x^2
f = function([X, y], [gradients[W].sum(), clipped_gradients[W].sum()],
allow_input_downcast=True)
# df/dW = df/db = 20 for W = 10, x = 1, so the norm is 20 * sqrt(2)
# and the gradients should be clipped to 20 / sqrt(2)
np.testing.assert_allclose(f([[1]], [[0]]), [20, 20 / np.sqrt(2)])
def test_conditional_initialize_parameters():
"""
Conditional.initialize_parameters does the following:
* Set its input_space and ndim attributes
* Calls its MLP's set_mlp method
* Sets its MLP's input_space
* Validates its MLP
* Sets its params and param names
"""
mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01,
max_col_norm=0.01)])
conditional = DummyConditional(mlp=mlp, name='conditional')
vae = DummyVAE()
conditional.set_vae(vae)
input_space = VectorSpace(dim=5)
conditional.initialize_parameters(input_space=input_space, ndim=5)
testing.assert_same_object(input_space, conditional.input_space)
testing.assert_equal(conditional.ndim, 5)
testing.assert_same_object(mlp.get_mlp(), conditional)
testing.assert_same_object(mlp.input_space, input_space)
mlp_params = mlp.get_params()
conditional_params = conditional.get_params()
assert all([mp in conditional_params for mp in mlp_params])
assert all([cp in mlp_params for cp in conditional_params])
def create_output_layer(self):
sc = 1. / self.n_classes
output = Linear(layer_name='tagger_out',
istdev=1. / sqrt(self.n_classes),
dim=self.n_classes,
W_lr_scale=sc,
b_lr_scale=sc)
return output
def test_conditional_get_vae():
"""
Conditional.get_vae returns its VAE
"""
mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01)])
conditional = DummyConditional(mlp=mlp, name='conditional')
vae = DummyVAE()
conditional.set_vae(vae)
testing.assert_same_object(conditional.get_vae(), vae)
def test_conditional_raises_exception_if_called_twice():
"""
Conditional.set_vae raises an exception if it has already been called
"""
mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01)])
conditional = DummyConditional(mlp=mlp, name='conditional')
vae = DummyVAE()
conditional.set_vae(vae)
conditional.set_vae(vae)
def test_conditional_returns_mlp_weights():
"""
Conditional.get_weights calls its MLP's get_weights method
"""
mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01)])
conditional = DummyConditional(mlp=mlp, name='conditional')
vae = DummyVAE()
conditional.set_vae(vae)
conditional.initialize_parameters(input_space=VectorSpace(dim=5), ndim=5)
numpy.testing.assert_equal(conditional.get_weights(), mlp.get_weights())
def get_Linear_Layer(self, structure, i=0):
n_input, n_output = structure
config = {
'dim': n_output,
'layer_name': ("l%d" % i),
'irange': .5,
'use_abs_loss': False,
'use_bias': False,
}
return Linear(**config)
def test_summarize_model():
"""
Asks the summarize_model.py script to inspect a pickled model and
check that it completes succesfully
"""
skip_if_no_matplotlib()
with open('model.pkl', 'wb') as f:
cPickle.dump(MLP(layers=[Linear(dim=5, layer_name='h0', irange=0.1)],
nvis=10), f, protocol=cPickle.HIGHEST_PROTOCOL)
summarize('model.pkl')
os.remove('model.pkl')
def test_diagonal_gaussian_default_output_layer():
"""
DiagonalGaussian's default output layer is compatible with its required
output space
"""
mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01,
max_col_norm=0.01)])
conditional = DiagonalGaussian(mlp=mlp, name='conditional')
vae = DummyVAE()
conditional.set_vae(vae)
input_space = VectorSpace(dim=5)
conditional.initialize_parameters(input_space=input_space, ndim=5)
def test_diagonal_gaussian_sample_from_epsilon():
"""
DiagonalGaussian.sample_from_epsilon doesn't crash
"""
mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01,
max_col_norm=0.01)])
conditional = DiagonalGaussian(mlp=mlp, name='conditional')
vae = DummyVAE()
conditional.set_vae(vae)
input_space = VectorSpace(dim=5)
conditional.initialize_parameters(input_space=input_space, ndim=5)
conditional.sample_from_epsilon((2, 10, 5))
def test_conditional_set_vae():
"""
Conditional.set_vae adds a reference to the vae and adopts the vae's rng
and batch_size attributes
"""
mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01)])
conditional = DummyConditional(mlp=mlp, name='conditional')
vae = DummyVAE()
conditional.set_vae(vae)
testing.assert_same_object(conditional.vae, vae)
testing.assert_same_object(conditional.rng, vae.rng)
testing.assert_equal(conditional.batch_size, vae.batch_size)
def test_bernoulli_vector_sample_from_conditional():
"""
BernoulliVector.sample_from_conditional works when num_samples is provided
"""
mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01,
max_col_norm=0.01)])
conditional = BernoulliVector(mlp=mlp, name='conditional')
vae = DummyVAE()
conditional.set_vae(vae)
input_space = VectorSpace(dim=5)
conditional.initialize_parameters(input_space=input_space, ndim=5)
mu = T.matrix('mu')
conditional.sample_from_conditional([mu], num_samples=2)
def test_bernoulli_vector_conditional_expectation():
"""
BernoulliVector.conditional_expectation doesn't crash
"""
mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01,
max_col_norm=0.01)])
conditional = BernoulliVector(mlp=mlp, name='conditional')
vae = DummyVAE()
conditional.set_vae(vae)
input_space = VectorSpace(dim=5)
conditional.initialize_parameters(input_space=input_space, ndim=5)
mu = T.matrix('mu')
conditional.conditional_expectation([mu])
def test_conditional_modify_updates():
"""
Conditional.modify_updates calls its MLP's modify_updates method
"""
mlp = MLP(layers=[Linear(layer_name='h', dim=5, irange=0.01,
max_col_norm=0.01)])
conditional = DummyConditional(mlp=mlp, name='conditional')
vae = DummyVAE()
conditional.set_vae(vae)
conditional.initialize_parameters(input_space=VectorSpace(dim=5), ndim=5)
updates = OrderedDict(zip(mlp.get_params(), mlp.get_params()))
testing.assert_equal(conditional.modify_updates(updates),
mlp.modify_updates(updates))
