def learning_rate(value=0.01): """ Wrap learning rate. """ return shared_scalar(value, name="learning_rate")
L2NORM_LIMIT = 1.9365 EPSILON = 1e-7 def clip_param_norm(): for param in model.parameters: if param.name.startswith("W"): l2_norms = np.sqrt(np.sum(param.get_value() ** 2, axis=0, keepdims=True)) desired_norms = np.clip(l2_norms, 0, L2NORM_LIMIT) scale = (desired_norms + EPSILON) / (l2_norms + EPSILON) param.set_value(param.get_value() * scale) if __name__ == '__main__': model = NeuralClassifier(input_dim=28 * 28) model.training_callbacks.append(clip_param_norm) model.stack(Dropout(0.2), Maxout(240, num_pieces=5, init=UniformInitializer(.005)), Maxout(240, num_pieces=5, init=UniformInitializer(.005)), Dense(10, 'linear', init=UniformInitializer(.005)), Softmax()) trainer = MomentumTrainer(model, {"learning_rate": shared_scalar(0.01), "momentum": 0.5}) annealer = ExponentialLearningRateAnnealer(trainer, debug=True) mnist = MiniBatches(MnistDataset(), batch_size=100) trainer.run(mnist, controllers=[annealer]) model.save_params(default_model)
""" An auto-encoder for compress MNIST images. """ import logging, os logging.basicConfig(level=logging.INFO) from deepy.dataset import MnistDataset, MiniBatches from deepy.networks import AutoEncoder from deepy.layers import Dense from deepy.trainers import SGDTrainer, LearningRateAnnealer from deepy.utils import shared_scalar model_path = os.path.join(os.path.dirname(__file__), "models", "mnist_autoencoder.gz") if __name__ == '__main__': model = AutoEncoder(input_dim=28 * 28, rep_dim=30) model.stack_encoders(Dense(50, 'tanh'), Dense(30)) model.stack_decoders(Dense(50, 'tanh'), Dense(28 * 28)) trainer = SGDTrainer(model, { 'learning_rate': shared_scalar(0.05), 'gradient_clipping': 3 }) mnist = MiniBatches(MnistDataset(for_autoencoder=True), batch_size=20) trainer.run(mnist, controllers=[LearningRateAnnealer(trainer)]) model.save_params(model_path)
#!/usr/bin/env python # -*- coding: utf-8 -*- """ An auto-encoder for compress MNIST images. """ import logging, os logging.basicConfig(level=logging.INFO) from deepy.dataset import MnistDataset, MiniBatches from deepy.networks import AutoEncoder from deepy.layers import Dense from deepy.trainers import SGDTrainer, LearningRateAnnealer from deepy.utils import shared_scalar model_path = os.path.join(os.path.dirname(__file__), "models", "mnist_autoencoder.gz") if __name__ == '__main__': model = AutoEncoder(input_dim=28 * 28, rep_dim=30) model.stack_encoders(Dense(50, 'tanh'), Dense(30)) model.stack_decoders(Dense(50, 'tanh'), Dense(28 * 28)) trainer = SGDTrainer(model, {'learning_rate': shared_scalar(0.05), 'gradient_clipping': 3}) mnist = MiniBatches(MnistDataset(for_autoencoder=True), batch_size=20) trainer.run(mnist, controllers=[LearningRateAnnealer(trainer)]) model.save_params(model_path)
def clip_param_norm(): for param in model.parameters: if param.name.startswith("W"): l2_norms = np.sqrt( np.sum(param.get_value()**2, axis=0, keepdims=True)) desired_norms = np.clip(l2_norms, 0, L2NORM_LIMIT) scale = (desired_norms + EPSILON) / (l2_norms + EPSILON) param.set_value(param.get_value() * scale) if __name__ == '__main__': model = NeuralClassifier(input_dim=28 * 28) model.training_callbacks.append(clip_param_norm) model.stack(Dropout(0.2), Maxout(240, num_pieces=5, init=UniformInitializer(.005)), Maxout(240, num_pieces=5, init=UniformInitializer(.005)), Dense(10, 'linear', init=UniformInitializer(.005)), Softmax()) trainer = MomentumTrainer(model, { "learning_rate": shared_scalar(0.01), "momentum": 0.5 }) annealer = ExponentialLearningRateAnnealer(debug=True) mnist = MiniBatches(MnistDataset(), batch_size=100) trainer.run(mnist, controllers=[annealer]) model.save_params(default_model)