def test_compute_norms_type_raises(): from lasagne.utils import compute_norms array = [[1, 2], [3, 4]] with pytest.raises(RuntimeError) as excinfo: compute_norms(array) assert ("Unsupported type") in str(excinfo.value)
def test_compute_norms_ndim6_raises(): from lasagne.utils import compute_norms array = np.random.randn(1, 2, 3, 4, 5, 6).astype(theano.config.floatX) with pytest.raises(ValueError) as excinfo: compute_norms(array) assert "Unsupported tensor dimensionality" in str(excinfo.value)
def test_compute_norms(): from lasagne.utils import compute_norms # Test numpy version of compute_norms array = np.random.randn(10, 20, 30, 40).astype(theano.config.floatX) norms = compute_norms(array) assert array.dtype == norms.dtype assert norms.shape[0] == array.shape[0] # Test theano version of compute_norms t_array = theano.shared(array) t_norms = compute_norms(t_array) # Check if they do not differ much assert np.allclose(t_norms.eval(), norms)
def test_compute_norms_axes(): from lasagne.utils import compute_norms # Test numpy versions of compute norms with axes array = np.random.randn(10, 20, 30, 40).astype(theano.config.floatX) norms = compute_norms(array, norm_axes=(0, 2)) assert array.dtype == norms.dtype assert norms.shape == (array.shape[1], array.shape[3]) # Test theano version of compute_norms t_array = theano.shared(array) t_norms = compute_norms(t_array, norm_axes=(0, 2)) # Check if they do not differ much assert np.allclose(t_norms.eval(), norms)
def test_compute_norms_axes(): from lasagne.utils import compute_norms array = np.random.randn(10, 20, 30, 40).astype(theano.config.floatX) norms = compute_norms(array, norm_axes=(0, 2)) assert array.dtype == norms.dtype assert norms.shape == (array.shape[1], array.shape[3])
def test_compute_norms(): from lasagne.utils import compute_norms array = np.random.randn(10, 20, 30, 40).astype(theano.config.floatX) norms = compute_norms(array) assert array.dtype == norms.dtype assert norms.shape[0] == array.shape[0]
def test_compute_norms_ndim1(): from lasagne.utils import compute_norms # Test numpy versions of compute norms with axes array = np.random.randn(10, ).astype(theano.config.floatX) norms = compute_norms(array) assert array.dtype == norms.dtype assert norms.shape == array.shape # Check if they do not differ much assert np.allclose(norms, abs(array)) # Test theano version of compute_norms t_array = theano.shared(array) t_norms = compute_norms(t_array) # Check if they do not differ much assert np.allclose(t_norms.eval(), norms)
def test_norm_constraint(ndim): import numpy as np import theano from lasagne.updates import norm_constraint from lasagne.utils import compute_norms max_norm = 0.01 param = theano.shared(np.random.randn(*((25,) * ndim)).astype(theano.config.floatX)) update = norm_constraint(param, max_norm) apply_update = theano.function([], [], updates=[(param, update)]) apply_update() assert param.dtype == update.dtype assert np.max(compute_norms(param.get_value())) <= max_norm * (1 + PCT_TOLERANCE)
def test_norm_constraint(ndim): import numpy as np import theano from lasagne.updates import norm_constraint from lasagne.utils import compute_norms max_norm = 0.01 param = theano.shared( np.random.randn(*((25, ) * ndim)).astype(theano.config.floatX)) update = norm_constraint(param, max_norm) apply_update = theano.function([], [], updates=[(param, update)]) apply_update() assert param.dtype == update.dtype assert (np.max(compute_norms(param.get_value())) <= max_norm * (1 + PCT_TOLERANCE))
def test_norm_constraint_norm_axes(): import numpy as np import theano from lasagne.updates import norm_constraint from lasagne.utils import compute_norms max_norm = 0.01 norm_axes = (0, 2) param = theano.shared( np.random.randn(10, 20, 30, 40).astype(theano.config.floatX) ) update = norm_constraint(param, max_norm, norm_axes=norm_axes) apply_update = theano.function([], [], updates=[(param, update)]) apply_update() assert param.dtype == update.dtype assert (np.max(compute_norms(param.get_value(), norm_axes=norm_axes)) <= max_norm*(1 + PCT_TOLERANCE))
import theano.tensor as T import lasagne from lasagne.utils import compute_norms from theano.printing import debugprint param = theano.shared(np.random.randn(3, 5).astype(theano.config.floatX)) print "\nparam" print param.get_value() update = param + 100 print "\nupdate" debugprint(update, print_type = True) update = lasagne.updates.norm_constraint(update, 10) print "\nnorm_constraint" debugprint(update, print_type = True) func = theano.function([], [], updates=[(param, update)]) # Apply constrained update _ = func() norms = compute_norms(param.get_value()) print "\nparam" param_value = param.get_value() # print compute_norms(param_value).shape print param_value print np.isclose(np.max(norms), 10)
import theano.tensor as T import lasagne from lasagne.utils import compute_norms from theano.printing import debugprint param = theano.shared(np.random.randn(3, 5).astype(theano.config.floatX)) print "\nparam" print param.get_value() update = param + 100 print "\nupdate" debugprint(update, print_type=True) update = lasagne.updates.norm_constraint(update, 10) print "\nnorm_constraint" debugprint(update, print_type=True) func = theano.function([], [], updates=[(param, update)]) # Apply constrained update _ = func() norms = compute_norms(param.get_value()) print "\nparam" param_value = param.get_value() # print compute_norms(param_value).shape print param_value print np.isclose(np.max(norms), 10)
def train_setup(): # Prepare Theano variables for inputs and targets input_var = T.tensor4('inputs') target_var = T.ivector('targets') # Create neural network model (depending on first command line parameter) print("Building model and compiling functions...") print( " with input dimension {0},{1},{2}".format( config.image_height, \ config.image_width, \ config. image_channel ) ) network = cnn_archi( input_var, \ config.image_channel,\ config.image_height, config.image_width,\ config.output_length ) print('Number of parameters : {0}'.format(count_params(network))) if (config.init_model is not None): with np.load(config.init_model) as f: param_values = [f['arr_%d' % i] for i in range(len(f.files))] set_all_param_values(network, param_values) # Create a loss expression for training, i.e., a scalar objective we want # to minimize (for our multi-class problem, it is the cross-entropy loss): prediction = lasagne.layers.get_output(network) ent_loss = categorical_crossentropy(prediction, target_var) ent_loss = ent_loss.mean() l1_regu = config.l1_regu * regularize_network_params(network, l1) l2_regu = config.l2_regu * regularize_network_params(network, l2) loss = ent_loss + l1_regu + l2_regu # We could add some weight decay as well here, see lasagne.regularization. # Create update expressions for training, i.e., how to modify the # parameters at each training step. Here, we'll use Stochastic Gradient # Descent (SGD) with Nesterov momentum, but Lasagne offers plenty more. params = get_all_params(network, trainable=True) #grads = T.grad( loss, params ) #scaled_grads = norm_constraint( grads, 5. ) updates = nesterov_momentum(loss, params, \ learning_rate=config.learning_rate, \ momentum=config.momentum ) #updates = rmsprop( loss , params, learning_rate = config.learning_rate ) for param in get_all_params(network, regularizable=True): norm_axis = None if param.ndim == 1: norm_axis = [0] updates[param] = norm_constraint( updates[param], \ 5. * compute_norms( param.get_value() ).mean(), norm_axes = norm_axis ) #Create a loss expression for validation/testing. The crucial difference # here is that we do a deterministic forward pass through the network, # disabling dropout layers. test_prediction = get_output(network, deterministic=True) test_classes = T.argmax(test_prediction, axis=1) test_loss = categorical_crossentropy(test_prediction, target_var) test_loss = test_loss.mean() # As a bonus, also create an expression for the classification accuracy: test_acc = T.eq(test_classes, target_var) # Compile a function performing a training step on a mini-batch (by giving # the updates dictionary) and returning the corresponding training loss: train_fn = theano.function([input_var,target_var], \ ent_loss,\ updates=updates, \ allow_input_downcast=True) # Compile a second function computing the validation loss and accuracy: val_fn = theano.function([input_var, target_var], \ [test_loss, test_prediction, test_acc], \ allow_input_downcast=True ) return network, train_fn, val_fn