def run():
# Prepare data
dataset = dp.datasets.MNIST()
x, y = dataset.data(flat=True)
x = x.astype(dp.float_)
y = y.astype(dp.int_)
train_idx, test_idx = dataset.split()
x_train = x[train_idx]
y_train = y[train_idx]
x_test = x[test_idx]
y_test = y[test_idx]
scaler = dp.UniformScaler(high=255.)
x_train = scaler.fit_transform(x_train)
x_test = scaler.transform(x_test)
batch_size = 128
train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size)
test_input = dp.SupervisedInput(x_test, y_test)
# Setup neural network
net = dp.NeuralNetwork(
layers=[
dp.FullyConnected(
n_output=800,
weights=dp.Parameter(dp.AutoFiller(), weight_decay=0.0001),
),
dp.Activation('relu'),
dp.FullyConnected(
n_output=800,
weights=dp.Parameter(dp.AutoFiller(), weight_decay=0.0001),
),
dp.Activation('relu'),
dp.FullyConnected(
n_output=dataset.n_classes,
weights=dp.Parameter(dp.AutoFiller(), weight_decay=0.0001),
),
dp.MultinomialLogReg(),
],
)
# Train neural network
def val_error():
return net.error(test_input)
trainer = dp.StochasticGradientDescent(
max_epochs=25,
learn_rule=dp.Momentum(learn_rate=0.1, momentum=0.9),
)
trainer.train(net, train_input, val_error)
# Visualize weights from first layer
W = next(np.array(layer.params()[0].array) for layer in net.layers
if isinstance(layer, dp.FullyConnected))
W = np.reshape(W.T, (-1, 28, 28))
filepath = os.path.join('mnist', 'mlp_weights.png')
dp.misc.img_save(dp.misc.img_tile(dp.misc.img_stretch(W)), filepath)
# Evaluate on test data
error = net.error(test_input)
print('Test error rate: %.4f' % error)
def run():
# Prepare data
dataset = dp.datasets.MNIST()
x, y = dataset.data(flat=True)
x = x.astype(dp.float_)/255.0
y = y.astype(dp.int_)
train_idx, test_idx = dataset.split()
x_train = x[train_idx]
y_train = y[train_idx]
x_test = x[test_idx]
y_test = y[test_idx]
train_input = dp.SupervisedInput(x_train, y_train, batch_size=128)
test_input = dp.SupervisedInput(x_test, y_test)
# Setup neural network
nn = dp.NeuralNetwork(
layers=[
dp.Dropout(0.2),
dp.DropoutFullyConnected(
n_output=800,
dropout=0.5,
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
penalty=('l2', 0.00001), monitor=True),
),
dp.Activation('relu'),
dp.DropoutFullyConnected(
n_output=800,
dropout=0.5,
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
penalty=('l2', 0.00001), monitor=True),
),
dp.Activation('relu'),
dp.DropoutFullyConnected(
n_output=dataset.n_classes,
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
penalty=('l2', 0.00001), monitor=True),
),
dp.MultinomialLogReg(),
],
)
# Train neural network
def valid_error():
return nn.error(test_input)
trainer = dp.StochasticGradientDescent(
max_epochs=50,
learn_rule=dp.Momentum(learn_rate=0.1, momentum=0.9),
)
trainer.train(nn, train_input, valid_error)
# Visualize weights from first layer
W = next(np.array(layer.params()[0].values) for layer in nn.layers
if isinstance(layer, dp.FullyConnected))
W = np.reshape(W.T, (-1, 28, 28))
dp.misc.img_save(dp.misc.img_tile(dp.misc.img_stretch(W)),
os.path.join('mnist', 'mlp_dropout_weights.png'))
# Evaluate on test data
error = nn.error(test_input)
print('Test error rate: %.4f' % error)
def test_classification():
# Make dataset
n_classes = 2
n_samples = 1000
n_features = 48
x, y = make_classification(n_samples=n_samples,
n_features=n_features,
n_classes=n_classes,
n_informative=n_classes * 2,
random_state=1)
x = x.astype(dp.float_)
y = y.astype(dp.int_)
n_train = int(0.8 * n_samples)
x_train = x[:n_train]
y_train = y[:n_train]
x_test = x[n_train:]
y_test = y[n_train:]
scaler = dp.StandardScaler()
x_train = scaler.fit_transform(x_train)
x_test = scaler.transform(x_test)
# Setup feeds
batch_size = 16
train_feed = dp.SupervisedFeed(x_train, y_train, batch_size=batch_size)
test_feed = dp.Feed(x_test)
# Setup neural network
weight_decay = 1e-03
net = dp.NeuralNetwork(
layers=[
dp.Affine(
n_out=32,
weights=dp.Parameter(dp.AutoFiller(),
weight_decay=weight_decay),
),
dp.ReLU(),
dp.Affine(
n_out=64,
weights=dp.Parameter(dp.AutoFiller(),
weight_decay=weight_decay),
),
dp.ReLU(),
dp.Affine(
n_out=n_classes,
weights=dp.Parameter(dp.AutoFiller()),
),
],
loss=dp.SoftmaxCrossEntropy(),
)
# Train neural network
learn_rule = dp.Momentum(learn_rate=0.01 / batch_size, momentum=0.9)
trainer = dp.GradientDescent(net, train_feed, learn_rule)
trainer.train_epochs(n_epochs=10)
# Evaluate on test data
error = np.mean(net.predict(test_feed) != y_test)
print('Test error rate: %.4f' % error)
assert error < 0.2
def test_network(model):
seq_size = 1
batch_size = 1
recurrent_nodes, fc_out = model
n_classes = fc_out.n_out
recurrent_graph = RecurrentGraph(
recurrent_nodes=recurrent_nodes, seq_size=seq_size,
batch_size=batch_size, cyclic=True, dropout=0.5
)
net = dp.NeuralNetwork(
layers=[
OneHot(n_classes=n_classes),
Reshape((seq_size, batch_size, -1)),
recurrent_graph,
Reshape((seq_size*batch_size, -1)),
fc_out,
dp.Softmax(),
],
loss=dp.loss.Loss(),
)
net._setup(x_shape=(seq_size,))
net.phase = 'test'
return net
def train_network(model, x_train, n_epochs=1000, learn_rate=0.2, batch_size=64,
seq_size=50, epoch_size=100):
recurrent_nodes, fc_out = model
n_classes = fc_out.n_out
recurrent_graph = RecurrentGraph(
recurrent_nodes=recurrent_nodes, seq_size=seq_size,
batch_size=batch_size, cyclic=True, dropout=0.5
)
net = dp.NeuralNetwork(
layers=[
OneHot(n_classes=n_classes),
Reshape((seq_size, batch_size, -1)),
recurrent_graph,
Reshape((seq_size*batch_size, -1)),
fc_out,
],
loss=dp.SoftmaxCrossEntropy(),
)
net.phase = 'train'
# Prepare network inputs
train_input = SupervisedSequenceInput(
x_train, seq_size=seq_size, batch_size=batch_size,
epoch_size=epoch_size
)
# Train network
try:
trainer = dp.StochasticGradientDescent(
max_epochs=n_epochs, min_epochs=n_epochs,
learn_rule=dp.RMSProp(learn_rate=learn_rate),
)
test_error = None
trainer.train(net, train_input, test_error)
except KeyboardInterrupt:
pass
return recurrent_nodes, fc_out
weight_gain_fc = 1.84
weight_decay_fc = 0.002
net = dp.NeuralNetwork(
layers=[
conv_layer(32),
dp.ReLU(),
pool_layer(),
conv_layer(64),
dp.ReLU(),
pool_layer(),
dp.Flatten(),
dp.Dropout(),
dp.Affine(
n_out=512,
weights=dp.Parameter(dp.AutoFiller(weight_gain_fc),
weight_decay=weight_decay_fc),
),
dp.ReLU(),
dp.Affine(
n_out=dataset.n_classes,
weights=dp.Parameter(dp.AutoFiller(weight_gain_fc)),
),
],
loss=dp.SoftmaxCrossEntropy(),
)
# Train network
n_epochs = [50, 15, 15]
learn_rate = 0.05 / batch_size
net = dp.NeuralNetwork(layers=[
dp.Convolutional(
n_filters=16,
filter_shape=(8, 8),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.0001),
weight_decay=0.004,
monitor=False),
strides=(4, 4),
),
dp.Activation('relu'),
dp.Convolutional(
n_filters=32,
filter_shape=(4, 4),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
weight_decay=0.004,
monitor=False),
strides=(2, 2),
),
dp.Activation('relu'),
dp.Convolutional(
n_filters=64,
filter_shape=(3, 3),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
weight_decay=0.004,
monitor=False),
strides=(1, 1),
),
dp.Activation('relu'),
dp.Flatten(),
dp.FullyConnected(
n_output=512,
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
weight_decay=0.004,
monitor=False),
),
dp.Activation('relu'),
dp.FullyConnected(
n_output=1,
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
weight_decay=0.004,
monitor=False),
),
dp.MeanSquaredError(),
], )
net = dp.NeuralNetwork(layers=[
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.0001),
weight_decay=0.004,
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
weight_decay=0.004,
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Convolutional(
n_filters=64,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
weight_decay=0.004,
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Flatten(),
dp.FullyConnected(
n_output=64,
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
weight_decay=0.004,
monitor=True),
),
dp.Activation('relu'),
dp.FullyConnected(
n_output=6,
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
weight_decay=0.004,
monitor=True),
),
dp.MeanSquaredError(),
], )
def run():
# Prepare data
dataset = dp.datasets.MNIST()
x, y = dataset.data()
x = x.astype(dp.float_)[:, np.newaxis, :, :]
y = y.astype(dp.int_)
train_idx, test_idx = dataset.split()
x_train = x[train_idx]
y_train = y[train_idx]
x_test = x[test_idx]
y_test = y[test_idx]
scaler = dp.UniformScaler(high=255.)
x_train = scaler.fit_transform(x_train)
x_test = scaler.transform(x_test)
batch_size = 128
train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size)
test_input = dp.SupervisedInput(x_test, y_test)
# Setup neural network
net = dp.NeuralNetwork(layers=[
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
weights=dp.Parameter(dp.AutoFiller(), weight_decay=0.0001),
),
dp.Activation('relu'),
dp.Pool(
win_shape=(3, 3),
strides=(2, 2),
method='max',
),
dp.Convolutional(
n_filters=64,
filter_shape=(5, 5),
weights=dp.Parameter(dp.AutoFiller(), weight_decay=0.0001),
),
dp.Activation('relu'),
dp.Pool(
win_shape=(3, 3),
strides=(2, 2),
method='max',
),
dp.Flatten(),
dp.FullyConnected(
n_output=128,
weights=dp.Parameter(dp.AutoFiller()),
),
dp.FullyConnected(
n_output=dataset.n_classes,
weights=dp.Parameter(dp.AutoFiller()),
),
dp.MultinomialLogReg(),
], )
# Train neural network
def val_error():
return net.error(test_input)
trainer = dp.StochasticGradientDescent(
max_epochs=15,
learn_rule=dp.Momentum(learn_rate=0.01, momentum=0.9),
)
trainer.train(net, train_input, val_error)
# Visualize convolutional filters to disk
for l, layer in enumerate(net.layers):
if not isinstance(layer, dp.Convolutional):
continue
W = np.array(layer.params()[0].array)
filepath = os.path.join('mnist', 'conv_layer_%i.png' % l)
dp.misc.img_save(dp.misc.conv_filter_tile(W), filepath)
# Evaluate on test data
error = net.error(test_input)
print('Test error rate: %.4f' % error)
def test_classification():
# Make dataset
n_classes = 2
n_samples = 1000
n_features = 48
x, y = make_classification(
n_samples=n_samples, n_features=n_features, n_classes=n_classes,
n_informative=n_classes*2, random_state=1
)
n_train = int(0.8 * n_samples)
n_val = int(0.5 * (n_samples - n_train))
x_train = x[:n_train]
y_train = y[:n_train]
x_val = x[n_train:n_train+n_val]
y_val = y[n_train:n_train+n_val]
x_test = x[n_train+n_val:]
y_test = y[n_train+n_val:]
scaler = dp.StandardScaler()
x_train = scaler.fit_transform(x_train)
x_val = scaler.transform(x_val)
x_test = scaler.transform(x_test)
# Setup input
batch_size = 16
train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size)
val_input = dp.Input(x_val)
test_input = dp.Input(x_test)
# Setup neural network
weight_decay = 1e-03
net = dp.NeuralNetwork(
layers=[
dp.Affine(
n_out=32,
weights=dp.Parameter(dp.AutoFiller(),
weight_decay=weight_decay),
),
dp.ReLU(),
dp.Affine(
n_out=64,
weights=dp.Parameter(dp.AutoFiller(),
weight_decay=weight_decay),
),
dp.ReLU(),
dp.Affine(
n_out=n_classes,
weights=dp.Parameter(dp.AutoFiller()),
),
],
loss=dp.SoftmaxCrossEntropy(),
)
# Train neural network
def val_error():
return np.mean(net.predict(val_input) != y_val)
trainer = dp.GradientDescent(
min_epochs=10, learn_rule=dp.Momentum(learn_rate=0.01, momentum=0.9),
)
trainer.train(net, train_input, val_error)
# Evaluate on test data
error = np.mean(net.predict(test_input) != y_test)
print('Test error rate: %.4f' % error)
assert error < 0.2
def run():
# Prepare data
batch_size = 128
dataset = dp.datasets.CIFAR10()
x, y = dataset.data()
y = y.astype(dp.int_)
train_idx, test_idx = dataset.split()
x_train = preprocess_imgs(x[train_idx])
y_train = y[train_idx]
train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size)
# Setup neural network
pool_kwargs = {
'win_shape': (3, 3),
'strides': (2, 2),
'border_mode': 'same',
'method': 'max',
}
nn = dp.NeuralNetwork(layers=[
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.0001),
penalty=('l2', 0.004),
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
penalty=('l2', 0.004),
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Convolutional(
n_filters=64,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
penalty=('l2', 0.004),
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Flatten(),
dp.FullyConnected(
n_output=64,
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
penalty=('l2', 0.03)),
),
dp.Activation('relu'),
dp.FullyConnected(
n_output=dataset.n_classes,
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
penalty=('l2', 0.03)),
),
dp.MultinomialLogReg(),
], )
dp.misc.profile(nn, train_input)
method='max',
)
net = dp.NeuralNetwork(
layers=[
conv_layer(32),
dp.ReLU(),
pool_layer(),
conv_layer(32),
dp.ReLU(),
pool_layer(),
conv_layer(64),
dp.ReLU(),
pool_layer(),
dp.Flatten(),
dp.Dropout(),
dp.Affine(n_out=64,
weights=dp.Parameter(dp.AutoFiller(gain=1.25),
weight_decay=0.03)),
dp.ReLU(),
dp.Affine(
n_out=dataset.n_classes,
weights=dp.Parameter(dp.AutoFiller(gain=1.25)),
)
],
loss=dp.SoftmaxCrossEntropy(),
)
profile(net, train_input)
weight_gain_fc = 1.84
weight_decay_fc = 0.002
net = dp.NeuralNetwork(
layers=[
conv_layer(32),
dp.Activation('relu'),
pool_layer(),
conv_layer(64),
dp.Activation('relu'),
pool_layer(),
dp.Flatten(),
dp.DropoutFullyConnected(
n_out=512,
dropout=0.5,
weights=dp.Parameter(dp.AutoFiller(weight_gain_fc),
weight_decay=weight_decay_fc),
),
dp.Activation('relu'),
dp.FullyConnected(
n_out=dataset.n_classes,
weights=dp.Parameter(dp.AutoFiller(weight_gain_fc)),
),
],
loss=dp.SoftmaxCrossEntropy(),
)
# Train network
n_epochs = [50, 15, 15]
learn_rate = 0.05
net = dp.NeuralNetwork(layers=[
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
weights=dp.Parameter(dp.AutoFiller(), weight_decay=0.0001),
),
dp.Activation('relu'),
dp.Pool(
win_shape=(3, 3),
strides=(2, 2),
method='max',
),
dp.Convolutional(
n_filters=64,
filter_shape=(5, 5),
weights=dp.Parameter(dp.AutoFiller(), weight_decay=0.0001),
),
dp.Activation('relu'),
dp.Pool(
win_shape=(3, 3),
strides=(2, 2),
method='max',
),
dp.Flatten(),
dp.FullyConnected(
n_output=128,
weights=dp.Parameter(dp.AutoFiller()),
),
dp.FullyConnected(
n_output=6,
weights=dp.Parameter(dp.AutoFiller()),
),
dp.MultinomialLogReg(),
], )
def run():
# Prepare data
dataset = dp.datasets.MNIST()
x, y = dataset.data()
x = x[:, np.newaxis, :, :].astype(dp.float_) / 255.0 - 0.5
y = y.astype(dp.int_)
train_idx, test_idx = dataset.split()
x_train = x[train_idx]
y_train = y[train_idx]
x_test = x[test_idx]
y_test = y[test_idx]
train_input = dp.SupervisedInput(x_train, y_train, batch_size=128)
test_input = dp.SupervisedInput(x_test, y_test)
# Setup neural network
nn = dp.NeuralNetwork(layers=[
dp.Convolutional(
n_filters=20,
filter_shape=(5, 5),
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
penalty=('l2', 0.00001)),
),
dp.Activation('relu'),
dp.Pool(
win_shape=(2, 2),
strides=(2, 2),
method='max',
),
dp.Convolutional(
n_filters=50,
filter_shape=(5, 5),
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
penalty=('l2', 0.00001)),
),
dp.Activation('relu'),
dp.Pool(
win_shape=(2, 2),
strides=(2, 2),
method='max',
),
dp.Flatten(),
dp.FullyConnected(
n_output=500,
weights=dp.NormalFiller(sigma=0.01),
),
dp.FullyConnected(
n_output=dataset.n_classes,
weights=dp.NormalFiller(sigma=0.01),
),
dp.MultinomialLogReg(),
], )
# Train neural network
def valid_error():
return nn.error(test_input)
trainer = dp.StochasticGradientDescent(
max_epochs=15,
learn_rule=dp.Momentum(learn_rate=0.1, momentum=0.9),
)
trainer.train(nn, train_input, valid_error)
# Visualize convolutional filters to disk
for layer_idx, layer in enumerate(nn.layers):
if not isinstance(layer, dp.Convolutional):
continue
W = np.array(layer.params()[0].values)
dp.misc.img_save(
dp.misc.conv_filter_tile(W),
os.path.join('mnist', 'convnet_layer_%i.png' % layer_idx))
# Evaluate on test data
error = nn.error(test_input)
print('Test error rate: %.4f' % error)
cor[i] = np.corrcoef(cnn_rep[:, i], pred_train[:, i])[0, 1]
svm_model = OneVsRestClassifier(LinearSVC(random_state=0))
svm_model.fit(cnn_rep[:, 0:numberOfModels], cnn_targets)
prediction = svm_model.predict(pred)
acc = np.sum(prediction == y_test) / float(np.size(y_test))
# Setup neural network using the stacked autoencoder layers
net = dp.NeuralNetwork(
[
dp.FullyConnected(
n_out=100, #neuronNum[-1],
weights=dp.Parameter(dp.AutoFiller()),
),
dp.Sigmoid(),
dp.FullyConnected(
n_out=10,
weights=dp.Parameter(dp.AutoFiller()),
),
],
loss=dp.loss.MeanSquaredError(),
)
# Fine-tune neural network
train_input = dp.SupervisedInput(brain_rep,
cnn_rep,
batch_size=batch_size)
test_input = dp.Input(x_test)
for i in range(len(epochs_value_pretraining)):
lr = learning_rate_pretraining / 10**i
trainer = dp.StochasticGradientDescent(
max_epochs=15,
learn_rule=dp.Momentum(learn_rate=0.05, momentum=0.9),
)
for ae in sae.ae_models():
trainer.train(ae, train_input)
# Train stacked autoencoders
trainer.train(sae, train_input)
# Setup neural network using the stacked autoencoder layers
net = dp.NeuralNetwork(
layers=sae.feedforward_layers() + [
dp.FullyConnected(
n_out=8,
# n_out=dataset.n_classes,
weights=dp.Parameter(dp.AutoFiller()),
),
],
loss=dp.SoftmaxCrossEntropy(),
)
# Fine-tune neural network
train_input = dp.SupervisedInput(x_train,
y_train,
batch_size=batch_size)
test_input = dp.Input(x_test)
trainer = dp.StochasticGradientDescent(
max_epochs=50,
learn_rule=dp.Momentum(learn_rate=0.05, momentum=0.9),
)
trainer.train(net, train_input)
def run():
# Prepare data
batch_size = 128
dataset = dp.datasets.CIFAR10()
x, y = dataset.data()
y = y.astype(dp.int_)
train_idx, test_idx = dataset.split()
x_train = preprocess_imgs(x[train_idx])
y_train = y[train_idx]
x_test = preprocess_imgs(x[test_idx])
y_test = y[test_idx]
train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size)
test_input = dp.SupervisedInput(x_test, y_test, batch_size=batch_size)
# Setup neural network
pool_kwargs = {
'win_shape': (3, 3),
'strides': (2, 2),
'border_mode': 'same',
'method': 'max',
}
nn = dp.NeuralNetwork(layers=[
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.0001),
penalty=('l2', 0.004),
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
penalty=('l2', 0.004),
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Convolutional(
n_filters=64,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
penalty=('l2', 0.004),
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Flatten(),
dp.FullyConnected(
n_output=64,
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
penalty=('l2', 0.03)),
),
dp.Activation('relu'),
dp.FullyConnected(
n_output=dataset.n_classes,
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
penalty=('l2', 0.03)),
),
dp.MultinomialLogReg(),
], )
# Train neural network
n_epochs = [8, 8]
learn_rate = 0.001
def valid_error():
return nn.error(test_input)
for i, max_epochs in enumerate(n_epochs):
lr = learn_rate / 10**i
trainer = dp.StochasticGradientDescent(
max_epochs=max_epochs,
learn_rule=dp.Momentum(learn_rate=lr, momentum=0.9),
)
trainer.train(nn, train_input, valid_error)
# Visualize convolutional filters to disk
for l, layer in enumerate(nn.layers):
if not isinstance(layer, dp.Convolutional):
continue
W = np.array(layer.params()[0].values)
dp.misc.img_save(dp.misc.conv_filter_tile(W),
os.path.join('cifar10', 'convnet_layer_%i.png' % l))
# Evaluate on test data
error = nn.error(test_input)
print('Test error rate: %.4f' % error)
border_mode='same',
method='max',
)
net = dp.NeuralNetwork(
layers=[
conv_layer(32),
dp.Activation('relu'),
pool_layer(),
conv_layer(32),
dp.Activation('relu'),
pool_layer(),
conv_layer(64),
dp.Activation('relu'),
pool_layer(),
dp.Flatten(),
dp.DropoutFullyConnected(n_out=64,
weights=dp.Parameter(dp.AutoFiller(gain=1.25),
weight_decay=0.03)),
dp.Activation('relu'),
dp.FullyConnected(
n_out=dataset.n_classes,
weights=dp.Parameter(dp.AutoFiller(gain=1.25)),
)
],
loss=dp.SoftmaxCrossEntropy(),
)
profile(net, train_input)
test_input = dp.Input(x_test)
# Setup network
weight_gain = 2.0
weight_decay = 0.0005
net = dp.NeuralNetwork(
layers=[
dp.FullyConnected(
n_out=1024,
weights=dp.Parameter(dp.AutoFiller(weight_gain),
weight_decay=weight_decay),
),
dp.ReLU(),
dp.FullyConnected(
n_out=1024,
weights=dp.Parameter(dp.AutoFiller(weight_gain),
weight_decay=weight_decay),
),
dp.ReLU(),
dp.FullyConnected(
n_out=dataset.n_classes,
weights=dp.Parameter(dp.AutoFiller()),
),
],
loss=dp.SoftmaxCrossEntropy(),
)
# Train network
n_epochs = [50, 15]
learn_rate = 0.05
for i, epochs in enumerate(n_epochs):
