def __init__(self, clipping_value, cost, exclude_params=[]):
assert isinstance(clipping_value, py_number_types)
assert isinstance(cost, Cost)
assert is_iterable(exclude_params)
assert all(isinstance(param_name, six.string_types)
for param_name in exclude_params)
self.__dict__.update(locals())
del self.self
def __init__(self, clipping_value, cost, exclude_params=[]):
assert isinstance(clipping_value, py_number_types)
assert isinstance(cost, Cost)
assert is_iterable(exclude_params)
assert all(
isinstance(param_name, six.string_types)
for param_name in exclude_params)
self.__dict__.update(locals())
del self.self
def _weight_decay_aggregate(self, method_name, coeff):
if isinstance(coeff, py_float_types):
return T.sum([getattr(layer, method_name)(coeff)
for layer in self.layers])
elif is_iterable(coeff):
assert all(layer_coeff >= 0 for layer_coeff in coeff)
return T.sum([getattr(layer, method_name)(layer_coeff) for
layer, layer_coeff in safe_zip(self.layers, coeff)
if layer_coeff > 0], dtype=config.floatX)
else:
raise TypeError("LayerList's " + method_name + " received "
"coefficients of type " + str(type(coeff)) + " "
"but must be provided with a float or list/tuple")
def get_weight_decay(self, coeff):
method_name = 'get_weight_decay'
if isinstance(coeff, py_float_types):
decay_list = [getattr(layer, method_name)(coeff)
for layer in self.layers]
elif is_iterable(coeff):
assert all(layer_coeff >= 0 for layer_coeff in coeff)
decay_list = [getattr(layer, method_name)(layer_coeff) for
layer, layer_coeff in safe_zip(self.layers, coeff)
if layer_coeff > 0]
else:
raise TypeError("CompositeLayer's " + method_name + " received "
"coefficients of type " + str(type(coeff)) + " "
"but must be provided with a float or list/tuple")
return decay_list
def _validate_impl(self, is_numeric, batch):
assert is_iterable(batch) and len(batch) == 2
self.data_space._validate_impl(is_numeric, batch[0])
self.mask_space._validate_impl(is_numeric, batch[1])
def test_composite_layer():
"""
Test the routing functionality of the CompositeLayer
"""
# Without routing
composite_layer = CompositeLayer('composite_layer',
[Linear(2, 'h0', irange=0),
Linear(2, 'h1', irange=0),
Linear(2, 'h2', irange=0)])
mlp = MLP(nvis=2, layers=[composite_layer])
for i in range(3):
composite_layer.layers[i].set_weights(
np.eye(2, dtype=theano.config.floatX)
)
composite_layer.layers[i].set_biases(
np.zeros(2, dtype=theano.config.floatX)
)
X = tensor.matrix()
y = mlp.fprop(X)
funs = [theano.function([X], y_elem) for y_elem in y]
x_numeric = np.random.rand(2, 2).astype('float32')
y_numeric = [f(x_numeric) for f in funs]
assert np.all(x_numeric == y_numeric)
# With routing
for inputs_to_layers in [{0: [1], 1: [2], 2: [0]},
{0: [1], 1: [0, 2], 2: []},
{0: [], 1: []}]:
composite_layer = CompositeLayer('composite_layer',
[Linear(2, 'h0', irange=0),
Linear(2, 'h1', irange=0),
Linear(2, 'h2', irange=0)],
inputs_to_layers)
input_space = CompositeSpace([VectorSpace(dim=2),
VectorSpace(dim=2),
VectorSpace(dim=2)])
mlp = MLP(input_space=input_space, layers=[composite_layer])
for i in range(3):
composite_layer.layers[i].set_weights(
np.eye(2, dtype=theano.config.floatX)
)
composite_layer.layers[i].set_biases(
np.zeros(2, dtype=theano.config.floatX)
)
X = [tensor.matrix() for _ in range(3)]
y = mlp.fprop(X)
funs = [theano.function(X, y_elem, on_unused_input='ignore')
for y_elem in y]
x_numeric = [np.random.rand(2, 2).astype(theano.config.floatX)
for _ in range(3)]
y_numeric = [f(*x_numeric) for f in funs]
assert all([all([np.all(x_numeric[i] == y_numeric[j])
for j in inputs_to_layers[i]])
for i in inputs_to_layers])
# Get the weight decay expressions from a composite layer
composite_layer = CompositeLayer('composite_layer',
[Linear(2, 'h0', irange=0.1),
Linear(2, 'h1', irange=0.1)])
input_space = VectorSpace(dim=10)
mlp = MLP(input_space=input_space, layers=[composite_layer])
for attr, coeff in product(['get_weight_decay', 'get_l1_weight_decay'],
[[0.7, 0.3], 0.5]):
f = theano.function([], getattr(composite_layer, attr)(coeff))
if is_iterable(coeff):
g = theano.function(
[], tensor.sum([getattr(layer, attr)(c) for c, layer
in zip(coeff, composite_layer.layers)])
)
assert np.allclose(f(), g())
else:
g = theano.function(
[], tensor.sum([getattr(layer, attr)(coeff) for layer
in composite_layer.layers])
)
assert np.allclose(f(), g())
def _validate_impl(self, is_numeric, batch):
assert is_iterable(batch) and len(batch) == 2
self.data_space._validate_impl(is_numeric, batch[0])
self.mask_space._validate_impl(is_numeric, batch[1])
