def run(method, model_path):
model = NeuralClassifier(input_dim=28 * 28)
model.stack(Dense(128, 'relu'),
Dense(128, 'relu'),
Dense(10, 'linear'),
Softmax())
trainer = ScipyTrainer(model, method)
annealer = LearningRateAnnealer()
mnist = MiniBatches(MnistDataset(), batch_size=100)
trainer.run(mnist, controllers=[annealer])
model.save_params(model_path)
def run(initializer, model_path):
model = NeuralClassifier(input_dim=28 * 28)
for _ in range(6):
model.stack(Dense(128, 'relu', init=initializer))
model.stack(Dense(10, 'linear'), Softmax())
trainer = MomentumTrainer(model)
annealer = LearningRateAnnealer(trainer)
mnist = MiniBatches(MnistDataset(), batch_size=20)
trainer.run(mnist, controllers=[annealer])
model.save_params(model_path)
def run(initializer, model_path):
model = NeuralClassifier(input_dim=28 * 28)
for _ in range(6):
model.stack(Dense(128, 'relu', init=initializer))
model.stack(Dense(10, 'linear'),
Softmax())
trainer = MomentumTrainer(model)
annealer = LearningRateAnnealer(trainer)
mnist = MiniBatches(MnistDataset(), batch_size=20)
trainer.run(mnist, controllers=[annealer])
model.save_params(model_path)
def run(method, model_path):
model = NeuralClassifier(input_dim=28 * 28)
model.stack(Dense(128, 'relu'), Dense(128, 'relu'), Dense(10, 'linear'),
Softmax())
trainer = ScipyTrainer(model, method)
annealer = LearningRateAnnealer()
mnist = MiniBatches(MnistDataset(), batch_size=100)
trainer.run(mnist, epoch_controllers=[annealer])
model.save_params(model_path)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging, os
logging.basicConfig(level=logging.INFO)
from deepy.dataset import MnistDataset, MiniBatches
from deepy.networks import NeuralClassifier
from deepy.layers import Dense, Softmax, Dropout, PRelu
from deepy.trainers import MomentumTrainer, LearningRateAnnealer
default_model = os.path.join(os.path.dirname(__file__), "models", "mlp_prelu_dropout1.gz")
if __name__ == '__main__':
model = NeuralClassifier(input_dim=28 * 28)
model.stack(Dense(256, 'linear'),
PRelu(),
Dropout(0.2),
Dense(256, 'linear'),
PRelu(),
Dropout(0.2),
Dense(10, 'linear'),
Softmax())
trainer = MomentumTrainer(model)
annealer = LearningRateAnnealer(trainer)
mnist = MiniBatches(MnistDataset(), batch_size=20)
trainer.run(mnist, controllers=[annealer])
default_model = os.path.join(os.path.dirname(__file__), "models", "mlp_maxout1.gz")
L2NORM_LIMIT = 1.9365
EPSILON = 1e-7
def clip_param_norm():
for param in model.parameters:
if param.name.startswith("W"):
l2_norms = np.sqrt(np.sum(param.get_value() ** 2, axis=0, keepdims=True))
desired_norms = np.clip(l2_norms, 0, L2NORM_LIMIT)
scale = (desired_norms + EPSILON) / (l2_norms + EPSILON)
param.set_value(param.get_value() * scale)
if __name__ == '__main__':
model = NeuralClassifier(input_dim=28 * 28)
model.training_callbacks.append(clip_param_norm)
model.stack(Dropout(0.2),
Maxout(240, num_pieces=5, init=UniformInitializer(.005)),
Maxout(240, num_pieces=5, init=UniformInitializer(.005)),
Dense(10, 'linear', init=UniformInitializer(.005)),
Softmax())
trainer = MomentumTrainer(model, {"learning_rate": shared_scalar(0.01),
"momentum": 0.5})
annealer = ExponentialLearningRateAnnealer(trainer, debug=True)
mnist = MiniBatches(MnistDataset(), batch_size=100)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging, os
logging.basicConfig(level=logging.INFO)
# MNIST Multi-layer model with dropout.
from deepy.dataset import MnistDataset, MiniBatches
from deepy.networks import NeuralClassifier
from deepy.layers import Dense, Softmax, Dropout
from deepy.trainers import MomentumTrainer, LearningRateAnnealer
model_path = os.path.join(os.path.dirname(__file__), "models", "tutorial1.gz")
if __name__ == '__main__':
model = NeuralClassifier(input_dim=28 * 28)
model.stack(Dense(256, 'relu'),
Dropout(0.2),
Dense(256, 'relu'),
Dropout(0.2),
Dense(10, 'linear'),
Softmax())
mnist = MiniBatches(MnistDataset(), batch_size=20)
trainer = MomentumTrainer(model, {"learning_rate": LearningRateAnnealer.learning_rate(0.01)})
annealer = LearningRateAnnealer(trainer)
trainer.run(mnist, controllers=[annealer])
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging, os
logging.basicConfig(level=logging.INFO)
from deepy.dataset import MnistDataset, MiniBatches
from deepy.networks import NeuralClassifier
from deepy.layers import Convolution, Dense, Flatten, DimShuffle, Reshape, RevealDimension, Softmax, Dropout
from deepy.trainers import MomentumTrainer, LearningRateAnnealer
default_model = os.path.join(os.path.dirname(__file__), "models",
"deep_conv.gz")
if __name__ == '__main__':
model = NeuralClassifier(input_dim=28 * 28)
model.stack( # Reshape to 3D tensor
Reshape((-1, 28, 28)),
# Add a new dimension for convolution
DimShuffle((0, 'x', 1, 2)),
Convolution((4, 1, 5, 5), activation="relu"),
Dropout(0.15),
Convolution((8, 4, 5, 5), activation="relu"),
Dropout(0.1),
Convolution((16, 8, 3, 3), activation="relu"),
Flatten(),
Dropout(0.1),
# As dimension information was lost, reveal it to the pipe line
RevealDimension(16),
Dense(10, 'linear'),
Softmax())
MNIST MLP model with batch normalization.
In my experiment, it turns out the improvement of valid data stopped after 37 epochs. (See models/batch_norm1.log)
"""
import logging, os
logging.basicConfig(level=logging.INFO)
from deepy.dataset import MnistDataset, MiniBatches
from deepy.networks import NeuralClassifier
from deepy.layers import Dense, Softmax, BatchNormalization
from deepy.trainers import SGDTrainer
default_model = os.path.join(os.path.dirname(__file__), "models", "batch_norm1.gz")
if __name__ == '__main__':
model = NeuralClassifier(input_dim=28*28)
model.stack(Dense(100, 'sigmoid'),
BatchNormalization(),
Dense(100, 'sigmoid'),
BatchNormalization(),
Dense(100, 'sigmoid'),
BatchNormalization(),
Dense(10, 'linear'),
Softmax())
trainer = SGDTrainer(model)
batches = MiniBatches(MnistDataset(), batch_size=60)
trainer.run(batches, controllers=[])
"""
This experiment setting is described in http://arxiv.org/pdf/1502.03167v3.pdf.
MNIST MLP baseline model.
Gaussian initialization described in the paper did not convergence, I have no idea.
"""
import logging, os
logging.basicConfig(level=logging.INFO)
from deepy.dataset import MnistDataset, MiniBatches
from deepy.networks import NeuralClassifier
from deepy.layers import Dense, Softmax
from deepy.trainers import SGDTrainer
default_model = os.path.join(os.path.dirname(__file__), "models", "baseline1.gz")
if __name__ == '__main__':
model = NeuralClassifier(input_dim=28 * 28)
model.stack(Dense(100, 'sigmoid'),
Dense(100, 'sigmoid'),
Dense(100, 'sigmoid'),
Dense(10, 'linear'),
Softmax())
trainer = SGDTrainer(model)
batches = MiniBatches(MnistDataset(), batch_size=60)
trainer.run(batches, epoch_controllers=[])
model.save_params(default_model)
http://arxiv.org/abs/1505.00387 .
With highway network layers, Very deep networks (20 layers here) can be trained properly.
"""
import logging, os
logging.basicConfig(level=logging.INFO)
from deepy.dataset import MnistDataset, MiniBatches
from deepy.networks import NeuralClassifier
from deepy.layers import Dense, Softmax
from deepy.trainers import MomentumTrainer, LearningRateAnnealer
from highway_layer import HighwayLayer
model_path = os.path.join(os.path.dirname(__file__), "models", "highway1.gz")
if __name__ == '__main__':
model = NeuralClassifier(input_dim=28 * 28)
model.stack(Dense(50, 'relu'))
for _ in range(20):
model.stack(HighwayLayer(activation='relu'))
model.stack(Dense(10, 'linear'),
Softmax())
trainer = MomentumTrainer(model)
mnist = MiniBatches(MnistDataset(), batch_size=20)
trainer.run(mnist, controllers=[LearningRateAnnealer()])
model.save_params(model_path)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging, os
logging.basicConfig(level=logging.INFO)
from deepy.dataset import MnistDataset, MiniBatches
from deepy.networks import NeuralClassifier
from deepy.layers import Convolution, Dense, Flatten, DimShuffle, Reshape, RevealDimension, Softmax, Dropout
from deepy.trainers import MomentumTrainer, LearningRateAnnealer
default_model = os.path.join(os.path.dirname(__file__), "models", "deep_conv.gz")
if __name__ == '__main__':
model = NeuralClassifier(input_dim=28*28)
model.stack(# Reshape to 3D tensor
Reshape((-1, 28, 28)),
# Add a new dimension for convolution
DimShuffle((0, 'x', 1, 2)),
Convolution((4, 1, 5, 5), activation="relu"),
Dropout(0.15),
Convolution((8, 4, 5, 5), activation="relu"),
Dropout(0.1),
Convolution((16, 8, 3, 3), activation="relu"),
Flatten(),
Dropout(0.1),
# As dimension information was lost, reveal it to the pipe line
RevealDimension(16),
Dense(10, 'linear'),
Softmax())
"""
This experiment setting is described in http://arxiv.org/pdf/1502.03167v3.pdf.
MNIST MLP baseline model.
Gaussian initialization described in the paper did not convergence, I have no idea.
"""
import logging, os
logging.basicConfig(level=logging.INFO)
from deepy.dataset import MnistDataset, MiniBatches
from deepy.networks import NeuralClassifier
from deepy.layers import Dense, Softmax
from deepy.trainers import SGDTrainer
default_model = os.path.join(os.path.dirname(__file__), "models", "baseline1.gz")
if __name__ == '__main__':
model = NeuralClassifier(input_dim=28 * 28)
model.stack(Dense(100, 'sigmoid'),
Dense(100, 'sigmoid'),
Dense(100, 'sigmoid'),
Dense(10, 'linear'),
Softmax())
trainer = SGDTrainer(model)
batches = MiniBatches(MnistDataset(), batch_size=60)
trainer.run(batches, controllers=[])
model.save_params(default_model)
# Shuffle the data
random.Random(3).shuffle(data)
# Separate data
valid_size = int(len(data) * 0.15)
train_set = data[valid_size:]
valid_set = data[:valid_size]
dataset = SequentialDataset(train_set, valid=valid_set)
dataset.pad_left(20)
dataset.report()
batch_set = MiniBatches(dataset)
if __name__ == '__main__':
model = NeuralClassifier(input_dim=26, input_tensor=3)
model.stack(RNN(hidden_size=30, input_type="sequence", output_type="sequence", vector_core=0.1),
RNN(hidden_size=30, input_type="sequence", output_type="sequence", vector_core=0.3),
RNN(hidden_size=30, input_type="sequence", output_type="sequence", vector_core=0.6),
RNN(hidden_size=30, input_type="sequence", output_type="one", vector_core=0.9),
Dense(4),
Softmax())
trainer = SGDTrainer(model)
annealer = LearningRateAnnealer(trainer)
trainer.run(batch_set.train_set(), batch_set.valid_set(), controllers=[annealer])
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
For reference, this model should achieve 1.50% error rate, in 10 mins with i7 CPU (8 threads).
"""
import logging, os
logging.basicConfig(level=logging.INFO)
from deepy.dataset import MnistDataset, MiniBatches
from deepy.networks import NeuralClassifier
from deepy.layers import Dense, Softmax, Dropout
from deepy.trainers import MomentumTrainer, LearningRateAnnealer
default_model = os.path.join(os.path.dirname(__file__), "models",
"mlp_dropout1.gz")
if __name__ == '__main__':
model = NeuralClassifier(input_dim=28 * 28)
model.stack(Dense(256, 'relu'), Dropout(0.5), Dense(256, 'relu'),
Dropout(0.5), Dense(10, 'linear'), Softmax())
trainer = MomentumTrainer(model)
annealer = LearningRateAnnealer()
mnist = MiniBatches(MnistDataset(), batch_size=20)
trainer.run(mnist, epoch_controllers=[annealer])
model.save_params(default_model)
Classify MNIST digits using a very deep think network.
Plain deep networks are very hard to be trained, as shown in this case.
But we should notice that if highway layers just learn to pass information forward,
in other words, just be transparent layers, then they would be meaningless.
"""
import logging, os
logging.basicConfig(level=logging.INFO)
from deepy.dataset import MnistDataset, MiniBatches
from deepy.networks import NeuralClassifier
from deepy.layers import Dense, Softmax
from deepy.trainers import MomentumTrainer, LearningRateAnnealer
model_path = os.path.join(os.path.dirname(__file__), "models", "baseline1.gz")
if __name__ == '__main__':
model = NeuralClassifier(input_dim=28 * 28)
for _ in range(20):
model.stack(Dense(71, 'relu'))
model.stack(Dense(10, 'linear'), Softmax())
trainer = MomentumTrainer(model)
mnist = MiniBatches(MnistDataset(), batch_size=20)
trainer.run(mnist, controllers=[LearningRateAnnealer()])
model.save_params(model_path)
Classify MNIST digits using a very deep think network.
Plain deep networks are very hard to be trained, as shown in this case.
But we should notice that if highway layers just learn to pass information forward,
in other words, just be transparent layers, then they would be meaningless.
"""
import logging, os
logging.basicConfig(level=logging.INFO)
from deepy.dataset import MnistDataset, MiniBatches
from deepy.networks import NeuralClassifier
from deepy.layers import Dense, Softmax
from deepy.trainers import MomentumTrainer, LearningRateAnnealer
model_path = os.path.join(os.path.dirname(__file__), "models", "baseline1.gz")
if __name__ == '__main__':
model = NeuralClassifier(input_dim=28 * 28)
for _ in range(20):
model.stack(Dense(71, 'relu'))
model.stack(Dense(10, 'linear'),
Softmax())
trainer = MomentumTrainer(model)
mnist = MiniBatches(MnistDataset(), batch_size=20)
trainer.run(mnist, controllers=[LearningRateAnnealer(trainer)])
model.save_params(model_path)
expanded_train_set = []
for img, label in mnist.train_set():
expanded_train_set.append((img, label))
original_img = (img * 256).reshape((28, 28))
transformed_img = (elastic_distortion(original_img) / 256).flatten()
expanded_train_set.append((transformed_img, label))
global_rand.shuffle(expanded_train_set)
expanded_mnist = BasicDataset(train=expanded_train_set, valid=mnist.valid_set(), test=mnist.test_set())
logging.info("expanded training data size: %d" % len(expanded_train_set))
if __name__ == '__main__':
model = NeuralClassifier(input_dim=28 * 28)
model.stack(Dense(256, 'relu'),
Dense(256, 'relu'),
Dense(10, 'linear'),
Softmax())
trainer = MomentumTrainer(model)
annealer = LearningRateAnnealer()
mnist = MiniBatches(expanded_mnist, batch_size=20)
trainer.run(mnist, controllers=[annealer])
model.save_params(default_model)
# Shuffle the data
random.Random(3).shuffle(data)
# Separate data
valid_size = int(len(data) * 0.15)
train_set = data[valid_size:]
valid_set = data[:valid_size]
dataset = SequentialDataset(train_set, valid=valid_set)
dataset.pad_left(20)
dataset.report()
batch_set = MiniBatches(dataset)
if __name__ == '__main__':
model = NeuralClassifier(input_dim=26, input_tensor=3)
model.stack(
RNN(hidden_size=30,
input_type="sequence",
output_type="sequence",
vector_core=0.1),
RNN(hidden_size=30,
input_type="sequence",
output_type="sequence",
vector_core=0.3),
RNN(hidden_size=30,
input_type="sequence",
output_type="sequence",
vector_core=0.6),
RNN(hidden_size=30,
input_type="sequence",
http://arxiv.org/abs/1505.00387 .
With highway network layers, Very deep networks (20 layers here) can be trained properly.
"""
import logging, os
logging.basicConfig(level=logging.INFO)
from deepy.dataset import MnistDataset, MiniBatches
from deepy.networks import NeuralClassifier
from deepy.layers import Dense, Softmax
from deepy.trainers import MomentumTrainer, LearningRateAnnealer
from highway_layer import HighwayLayer
model_path = os.path.join(os.path.dirname(__file__), "models", "highway1.gz")
if __name__ == '__main__':
model = NeuralClassifier(input_dim=28 * 28)
model.stack(Dense(50, 'relu'))
for _ in range(20):
model.stack(HighwayLayer(activation='relu'))
model.stack(Dense(10, 'linear'), Softmax())
trainer = MomentumTrainer(model)
mnist = MiniBatches(MnistDataset(), batch_size=20)
trainer.run(mnist, controllers=[LearningRateAnnealer()])
model.save_params(model_path)
import logging
import os
logging.basicConfig(level=logging.INFO)
from deepy.dataset import MiniBatches,BasicDataset
from deepy.networks import NeuralClassifier
from deepy.layers import Dense, Softmax
from deepy.trainers import MomentumTrainer, LearningRateAnnealer
from SynthDataset import SynthDataset
model_path = os.path.join(os.path.dirname(__file__), "models", "model_10000_op.gz")
if __name__ == '__main__':
model = NeuralClassifier(input_dim=32*32)
model.stack(Dense(400, 'tanh'),
Dense(100, 'tanh'),
Dense(3, 'linear'),
Softmax())
#trainer = MomentumTrainer(model, {"weight_l2": 0.01})
trainer = MomentumTrainer(model, {"learning_rate": LearningRateAnnealer.learning_rate(0.001)})
annealer = LearningRateAnnealer(trainer)
mlp_synthDataSet = MiniBatches(SynthDataset())
trainer.run(mlp_synthDataSet, controllers=[annealer])
model.save_params(model_path)