def run(method, model_path): model = NeuralClassifier(input_dim=28 * 28) model.stack(Dense(128, 'relu'), Dense(128, 'relu'), Dense(10, 'linear'), Softmax()) trainer = ScipyTrainer(model, method) annealer = LearningRateAnnealer() mnist = MiniBatches(MnistDataset(), batch_size=100) trainer.run(mnist, epoch_controllers=[annealer]) model.save_params(model_path)
def run(initializer, model_path): model = NeuralClassifier(input_dim=28 * 28) for _ in range(6): model.stack(Dense(128, 'relu', init=initializer)) model.stack(Dense(10, 'linear'), Softmax()) trainer = MomentumTrainer(model) annealer = LearningRateAnnealer(trainer) mnist = MiniBatches(MnistDataset(), batch_size=20) trainer.run(mnist, controllers=[annealer]) model.save_params(model_path)
from deepy.utils import Timer logging.basicConfig(level=logging.INFO) if __name__ == '__main__': ap = ArgumentParser() ap.add_argument("--model", default="/tmp/mnist_att_params2.gz") ap.add_argument("--method", default="momentum") ap.add_argument("--learning_rate", default=0.01) ap.add_argument("--variance", default=0.005) ap.add_argument("--disable_backprop", default=False) ap.add_argument("--disable_reinforce", default=False) ap.add_argument("--random_glimpse", default=False) args = ap.parse_args() mnist = MiniBatches((MnistDataset()), batch_size=1) model_path = args.model network = get_network(model_path, std=args.variance, disable_reinforce=args.disable_reinforce, random_glimpse=args.random_glimpse) trainer_conf = TrainerConfig() trainer_conf.learning_rate = args.learning_rate trainer_conf.weight_l2 = 0.0001 trainer_conf.hidden_l2 = 0.0001 trainer_conf.method = args.method trainer_conf.disable_reinforce=args.disable_reinforce trainer_conf.disable_backprop=args.disable_backprop trainer = AttentionTrainer(network, network.layers[0], config=trainer_conf)
default_model = os.path.join(os.path.dirname(__file__), "models", "deep_conv.gz") if __name__ == '__main__': model = NeuralClassifier(input_dim=28 * 28) model.stack( # Reshape to 3D tensor Reshape((-1, 28, 28)), # Add a new dimension for convolution DimShuffle((0, 'x', 1, 2)), Convolution((4, 1, 5, 5), activation="relu"), Dropout(0.15), Convolution((8, 4, 5, 5), activation="relu"), Dropout(0.1), Convolution((16, 8, 3, 3), activation="relu"), Flatten(), Dropout(0.1), # As dimension information was lost, reveal it to the pipe line RevealDimension(16), Dense(10, 'linear'), Softmax()) trainer = MomentumTrainer(model) annealer = LearningRateAnnealer() mnist = MiniBatches(MnistDataset(), batch_size=20) trainer.run(mnist, controllers=[annealer]) model.save_params(default_model)
""" An auto-encoder for compress MNIST images. """ import logging, os logging.basicConfig(level=logging.INFO) from deepy.dataset import MnistDataset, MiniBatches from deepy.networks import AutoEncoder from deepy.layers import Dense from deepy.trainers import SGDTrainer, LearningRateAnnealer from deepy.utils import shared_scalar model_path = os.path.join(os.path.dirname(__file__), "models", "mnist_autoencoder.gz") if __name__ == '__main__': model = AutoEncoder(input_dim=28 * 28, rep_dim=30) model.stack_encoders(Dense(50, 'tanh'), Dense(30)) model.stack_decoders(Dense(50, 'tanh'), Dense(28 * 28)) trainer = SGDTrainer(model, { 'learning_rate': shared_scalar(0.05), 'gradient_clipping': 3 }) mnist = MiniBatches(MnistDataset(for_autoencoder=True), batch_size=20) trainer.run(mnist, controllers=[LearningRateAnnealer(trainer)]) model.save_params(model_path)
""" import logging, os logging.basicConfig(level=logging.INFO) from deepy.dataset import MnistDataset, MiniBatches, BasicDataset from deepy.networks import NeuralClassifier from deepy.layers import Dense, Softmax from deepy.trainers import MomentumTrainer, LearningRateAnnealer from deepy.preprocessing import elastic_distortion from deepy.core import env default_model = os.path.join(os.path.dirname(__file__), "models", "mlp_distortion1.gz") mnist = MnistDataset() logging.info("transforming images with elastic distortion") expanded_train_set = [] for img, label in mnist.train_set(): expanded_train_set.append((img, label)) original_img = (img * 256).reshape((28, 28)) transformed_img = (elastic_distortion(original_img) / 256).flatten() expanded_train_set.append((transformed_img, label)) env.numpy_rand.shuffle(expanded_train_set) expanded_mnist = BasicDataset(train=expanded_train_set, valid=mnist.valid_set(), test=mnist.test_set())
better use multi-processing. """ import logging, os logging.basicConfig(level=logging.INFO) from deepy.dataset import MnistDataset, MiniBatches, BasicDataset from deepy.networks import NeuralClassifier from deepy.layers import Dense, Softmax from deepy.trainers import MomentumTrainer, LearningRateAnnealer from deepy.utils import elastic_distortion, global_rand default_model = os.path.join(os.path.dirname(__file__), "models", "mlp_distortion1.gz") mnist = MnistDataset() logging.info("transforming images with elastic distortion") expanded_train_set = [] for img, label in mnist.train_set(): expanded_train_set.append((img, label)) original_img = (img * 256).reshape((28, 28)) transformed_img = (elastic_distortion(original_img) / 256).flatten() expanded_train_set.append((transformed_img, label)) global_rand.shuffle(expanded_train_set) expanded_mnist = BasicDataset(train=expanded_train_set, valid=mnist.valid_set(),
""" This experiment setting is described in http://arxiv.org/pdf/1502.03167v3.pdf. MNIST MLP baseline model. Gaussian initialization described in the paper did not convergence, I have no idea. """ import logging, os logging.basicConfig(level=logging.INFO) from deepy.dataset import MnistDataset, MiniBatches from deepy.networks import NeuralClassifier from deepy.layers import Dense, Softmax from deepy.trainers import SGDTrainer default_model = os.path.join(os.path.dirname(__file__), "models", "baseline1.gz") if __name__ == '__main__': model = NeuralClassifier(input_dim=28 * 28) model.stack(Dense(100, 'sigmoid'), Dense(100, 'sigmoid'), Dense(100, 'sigmoid'), Dense(10, 'linear'), Softmax()) trainer = SGDTrainer(model) batches = MiniBatches(MnistDataset(), batch_size=60) trainer.run(batches, epoch_controllers=[]) model.save_params(default_model)