def create_weight(self, input_n=1, output_n=1, suffix="", initializer=None, shape=None):
if not shape:
shape = (input_n, output_n)
if not initializer:
initializer = UniformInitializer()
weight = theano.shared(initializer.sample(shape).astype(FLOATX), name='W_{}'.format(suffix))
logging.info('create weight W_%s: %s', suffix, str(shape))
return weight
def create_weight(self, input_n=1, output_n=1, suffix="W", initializer=None, shape=None):
if not shape:
shape = (input_n, output_n)
if not initializer:
initializer = UniformInitializer()
weight = theano.shared(initializer.sample(shape).astype(FLOATX), name='{}_{}'.format(self.name, suffix))
logging.info('create weight W_%s: %s', suffix, str(shape))
return weight
def _create_mlp(self):
model = NeuralRegressor(input_dim=self.unit_counts[0])
initializer = UniformInitializer(seed=self.random_state)
if self.spec.is_convolution:
model.stack_layer(layers.DimShuffle((0, 'x', 1, 2)))
for l, n in zip(self.layers, self.unit_counts[1:]):
t = None
if l.type in ('Tanh', 'Sigmoid'): t = l.type.lower()
if l.type in ('Rectifier'): t = 'relu'
if l.type in ('Linear', 'Softmax'): t = 'linear'
assert t is not None, "Unknown activation type `%s`." % l.type
if isinstance(l, Layer):
# self._check_layer(l, ['units'])
model.stack_layer(layers.Dense(n, t, init=initializer))
if l.type == 'Softmax':
model.stack_layer(layers.Softmax())
if isinstance(l, layers.Convolution):
# self._check_layer(l, ['channel', 'kernel_shape'])
model.stack_layer(layers.Convolution(
activation=t,
filter_shape=(l.channels, l.kernel_shape[0], l.kernel_shape[1]),
pool_size=l.pool_shape,
border_mode=l.border_mode,
init=initializer))
self.mlp = model
L2NORM_LIMIT = 1.9365
EPSILON = 1e-7
def clip_param_norm():
for param in model.parameters:
if param.name.startswith("W"):
l2_norms = np.sqrt(np.sum(param.get_value() ** 2, axis=0, keepdims=True))
desired_norms = np.clip(l2_norms, 0, L2NORM_LIMIT)
scale = (desired_norms + EPSILON) / (l2_norms + EPSILON)
param.set_value(param.get_value() * scale)
if __name__ == '__main__':
model = NeuralClassifier(input_dim=28 * 28)
model.training_callbacks.append(clip_param_norm)
model.stack(Dropout(0.2),
Maxout(240, num_pieces=5, init=UniformInitializer(.005)),
Maxout(240, num_pieces=5, init=UniformInitializer(.005)),
Dense(10, 'linear', init=UniformInitializer(.005)),
Softmax())
trainer = MomentumTrainer(model, {"learning_rate": shared_scalar(0.01),
"momentum": 0.5})
annealer = ExponentialLearningRateAnnealer(trainer, debug=True)
mnist = MiniBatches(MnistDataset(), batch_size=100)
trainer.run(mnist, controllers=[annealer])
model.save_params(default_model)
def _default_initializer(self):
fan_in = np.prod(self.filter_shape[1:])
fan_out = (self.filter_shape[0] * np.prod(self.filter_shape[2:]) /
np.prod(self.pool_size))
weight_scale = np.sqrt(6. / (fan_in + fan_out))
return UniformInitializer(scale=weight_scale)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
from util import run
from deepy.utils import UniformInitializer
model_path = os.path.join(os.path.dirname(__file__), "models", "uniform1.gz")
if __name__ == '__main__':
run(UniformInitializer(), model_path)