def test_compute_norms_type_raises():
from lasagne.utils import compute_norms
array = [[1, 2], [3, 4]]
with pytest.raises(RuntimeError) as excinfo:
compute_norms(array)
assert ("Unsupported type") in str(excinfo.value)
def test_compute_norms_type_raises():
from lasagne.utils import compute_norms
array = [[1, 2], [3, 4]]
with pytest.raises(RuntimeError) as excinfo:
compute_norms(array)
assert ("Unsupported type") in str(excinfo.value)
def test_compute_norms_ndim6_raises():
from lasagne.utils import compute_norms
array = np.random.randn(1, 2, 3, 4, 5, 6).astype(theano.config.floatX)
with pytest.raises(ValueError) as excinfo:
compute_norms(array)
assert "Unsupported tensor dimensionality" in str(excinfo.value)
def test_compute_norms_ndim6_raises():
from lasagne.utils import compute_norms
array = np.random.randn(1, 2, 3, 4, 5, 6).astype(theano.config.floatX)
with pytest.raises(ValueError) as excinfo:
compute_norms(array)
assert "Unsupported tensor dimensionality" in str(excinfo.value)
def test_compute_norms():
from lasagne.utils import compute_norms
# Test numpy version of compute_norms
array = np.random.randn(10, 20, 30, 40).astype(theano.config.floatX)
norms = compute_norms(array)
assert array.dtype == norms.dtype
assert norms.shape[0] == array.shape[0]
# Test theano version of compute_norms
t_array = theano.shared(array)
t_norms = compute_norms(t_array)
# Check if they do not differ much
assert np.allclose(t_norms.eval(), norms)
def test_compute_norms_axes():
from lasagne.utils import compute_norms
# Test numpy versions of compute norms with axes
array = np.random.randn(10, 20, 30, 40).astype(theano.config.floatX)
norms = compute_norms(array, norm_axes=(0, 2))
assert array.dtype == norms.dtype
assert norms.shape == (array.shape[1], array.shape[3])
# Test theano version of compute_norms
t_array = theano.shared(array)
t_norms = compute_norms(t_array, norm_axes=(0, 2))
# Check if they do not differ much
assert np.allclose(t_norms.eval(), norms)
def test_compute_norms_axes():
from lasagne.utils import compute_norms
array = np.random.randn(10, 20, 30, 40).astype(theano.config.floatX)
norms = compute_norms(array, norm_axes=(0, 2))
assert array.dtype == norms.dtype
assert norms.shape == (array.shape[1], array.shape[3])
def test_compute_norms():
from lasagne.utils import compute_norms
array = np.random.randn(10, 20, 30, 40).astype(theano.config.floatX)
norms = compute_norms(array)
assert array.dtype == norms.dtype
assert norms.shape[0] == array.shape[0]
def test_compute_norms_axes():
from lasagne.utils import compute_norms
array = np.random.randn(10, 20, 30, 40).astype(theano.config.floatX)
norms = compute_norms(array, norm_axes=(0, 2))
assert array.dtype == norms.dtype
assert norms.shape == (array.shape[1], array.shape[3])
def test_compute_norms():
from lasagne.utils import compute_norms
array = np.random.randn(10, 20, 30, 40).astype(theano.config.floatX)
norms = compute_norms(array)
assert array.dtype == norms.dtype
assert norms.shape[0] == array.shape[0]
def test_compute_norms_ndim1():
from lasagne.utils import compute_norms
# Test numpy versions of compute norms with axes
array = np.random.randn(10, ).astype(theano.config.floatX)
norms = compute_norms(array)
assert array.dtype == norms.dtype
assert norms.shape == array.shape
# Check if they do not differ much
assert np.allclose(norms, abs(array))
# Test theano version of compute_norms
t_array = theano.shared(array)
t_norms = compute_norms(t_array)
# Check if they do not differ much
assert np.allclose(t_norms.eval(), norms)
def test_norm_constraint(ndim):
import numpy as np
import theano
from lasagne.updates import norm_constraint
from lasagne.utils import compute_norms
max_norm = 0.01
param = theano.shared(np.random.randn(*((25,) * ndim)).astype(theano.config.floatX))
update = norm_constraint(param, max_norm)
apply_update = theano.function([], [], updates=[(param, update)])
apply_update()
assert param.dtype == update.dtype
assert np.max(compute_norms(param.get_value())) <= max_norm * (1 + PCT_TOLERANCE)
def test_norm_constraint(ndim):
import numpy as np
import theano
from lasagne.updates import norm_constraint
from lasagne.utils import compute_norms
max_norm = 0.01
param = theano.shared(
np.random.randn(*((25, ) * ndim)).astype(theano.config.floatX))
update = norm_constraint(param, max_norm)
apply_update = theano.function([], [], updates=[(param, update)])
apply_update()
assert param.dtype == update.dtype
assert (np.max(compute_norms(param.get_value())) <= max_norm *
(1 + PCT_TOLERANCE))
def test_norm_constraint_norm_axes():
import numpy as np
import theano
from lasagne.updates import norm_constraint
from lasagne.utils import compute_norms
max_norm = 0.01
norm_axes = (0, 2)
param = theano.shared(
np.random.randn(10, 20, 30, 40).astype(theano.config.floatX)
)
update = norm_constraint(param, max_norm, norm_axes=norm_axes)
apply_update = theano.function([], [], updates=[(param, update)])
apply_update()
assert param.dtype == update.dtype
assert (np.max(compute_norms(param.get_value(), norm_axes=norm_axes)) <=
max_norm*(1 + PCT_TOLERANCE))
import theano.tensor as T import lasagne from lasagne.utils import compute_norms from theano.printing import debugprint param = theano.shared(np.random.randn(3, 5).astype(theano.config.floatX)) print "\nparam" print param.get_value() update = param + 100 print "\nupdate" debugprint(update, print_type = True) update = lasagne.updates.norm_constraint(update, 10) print "\nnorm_constraint" debugprint(update, print_type = True) func = theano.function([], [], updates=[(param, update)]) # Apply constrained update _ = func() norms = compute_norms(param.get_value()) print "\nparam" param_value = param.get_value() # print compute_norms(param_value).shape print param_value print np.isclose(np.max(norms), 10)
import theano.tensor as T import lasagne from lasagne.utils import compute_norms from theano.printing import debugprint param = theano.shared(np.random.randn(3, 5).astype(theano.config.floatX)) print "\nparam" print param.get_value() update = param + 100 print "\nupdate" debugprint(update, print_type=True) update = lasagne.updates.norm_constraint(update, 10) print "\nnorm_constraint" debugprint(update, print_type=True) func = theano.function([], [], updates=[(param, update)]) # Apply constrained update _ = func() norms = compute_norms(param.get_value()) print "\nparam" param_value = param.get_value() # print compute_norms(param_value).shape print param_value print np.isclose(np.max(norms), 10)
def train_setup():
# Prepare Theano variables for inputs and targets
input_var = T.tensor4('inputs')
target_var = T.ivector('targets')
# Create neural network model (depending on first command line parameter)
print("Building model and compiling functions...")
print( " with input dimension {0},{1},{2}".format( config.image_height, \
config.image_width, \
config. image_channel ) )
network = cnn_archi( input_var, \
config.image_channel,\
config.image_height, config.image_width,\
config.output_length )
print('Number of parameters : {0}'.format(count_params(network)))
if (config.init_model is not None):
with np.load(config.init_model) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
set_all_param_values(network, param_values)
# Create a loss expression for training, i.e., a scalar objective we want
# to minimize (for our multi-class problem, it is the cross-entropy loss):
prediction = lasagne.layers.get_output(network)
ent_loss = categorical_crossentropy(prediction, target_var)
ent_loss = ent_loss.mean()
l1_regu = config.l1_regu * regularize_network_params(network, l1)
l2_regu = config.l2_regu * regularize_network_params(network, l2)
loss = ent_loss + l1_regu + l2_regu
# We could add some weight decay as well here, see lasagne.regularization.
# Create update expressions for training, i.e., how to modify the
# parameters at each training step. Here, we'll use Stochastic Gradient
# Descent (SGD) with Nesterov momentum, but Lasagne offers plenty more.
params = get_all_params(network, trainable=True)
#grads = T.grad( loss, params )
#scaled_grads = norm_constraint( grads, 5. )
updates = nesterov_momentum(loss, params, \
learning_rate=config.learning_rate, \
momentum=config.momentum )
#updates = rmsprop( loss , params, learning_rate = config.learning_rate )
for param in get_all_params(network, regularizable=True):
norm_axis = None
if param.ndim == 1:
norm_axis = [0]
updates[param] = norm_constraint( updates[param], \
5. * compute_norms( param.get_value() ).mean(),
norm_axes = norm_axis )
#Create a loss expression for validation/testing. The crucial difference
# here is that we do a deterministic forward pass through the network,
# disabling dropout layers.
test_prediction = get_output(network, deterministic=True)
test_classes = T.argmax(test_prediction, axis=1)
test_loss = categorical_crossentropy(test_prediction, target_var)
test_loss = test_loss.mean()
# As a bonus, also create an expression for the classification accuracy:
test_acc = T.eq(test_classes, target_var)
# Compile a function performing a training step on a mini-batch (by giving
# the updates dictionary) and returning the corresponding training loss:
train_fn = theano.function([input_var,target_var], \
ent_loss,\
updates=updates, \
allow_input_downcast=True)
# Compile a second function computing the validation loss and accuracy:
val_fn = theano.function([input_var, target_var], \
[test_loss, test_prediction, test_acc], \
allow_input_downcast=True )
return network, train_fn, val_fn
