def pool_layer(pool_method, border_mode):
return dp.Pool(
win_shape=(3, 3),
strides=(2, 2),
method=pool_method,
border_mode=border_mode,
)
def pool_layer():
return dp.Pool(
win_shape=(2, 2),
strides=(2, 2),
border_mode='valid',
method='max',
)
def run():
np.random.seed(3)
layers = [
dp.Activation('relu'),
dp.Activation('sigmoid'),
dp.Activation('tanh'),
dp.FullyConnected(
n_output=3,
weights=dp.NormalFiller(sigma=0.01),
),
dp.Dropout(0.2),
dp.DropoutFullyConnected(
n_output=10,
weights=dp.NormalFiller(sigma=0.01),
dropout=0.5,
),
]
input_shape = (1, 5)
x = np.random.normal(size=input_shape).astype(dp.float_)
for layer in layers:
dp.misc.check_bprop(layer, x)
conv_layers = [
dp.Convolutional(
n_filters=32,
filter_shape=(3, 3),
border_mode='same',
weights=dp.NormalFiller(sigma=0.01),
),
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
border_mode='valid',
weights=dp.NormalFiller(sigma=0.01),
),
dp.Pool(
win_shape=(3, 3),
strides=(2, 2),
method='max',
)
]
input_shape = (5, 3, 8, 8)
x = np.random.normal(size=input_shape).astype(dp.float_)
for layer in conv_layers:
dp.misc.check_bprop(layer, x)
def test_pool():
win_shapes = [(1, 1), (2, 2), (3, 3)]
strides = [(1, 1), (2, 2), (3, 3)]
border_modes = ['valid', 'same', 'full']
methods = ['max', 'avg']
confs = itertools.product(batch_sizes, n_channels, img_shapes,
win_shapes, strides, border_modes, methods)
# Sample random parameter configurations to reduce workload.
confs = shuffled(confs)[:100]
for (batch_size, n_channel, img_shape, win_shape, stride, border_mode,
method) in confs:
if img_shape[0] < win_shape[0] or img_shape[1] < win_shape[1]:
continue
if border_mode != 'valid' and \
(win_shape[0] != 1 or win_shape[1] != 1) and \
ca._backend == 'cuda' and \
ca.nnet.pool._default_impl == 'cudnn':
# Bug: I think CUDArray/DeepPy calculates the padding in a manner
# inconsistent with cuDNN
continue
if method == 'avg' and \
ca._backend == 'cuda' and \
ca.nnet.pool._default_impl == 'masked':
# Not implemented yet
continue
print('Pool: batch_size=%i, n_channel=%i, img_shape=%s, win_shape=%s, '
'stride=%s, border_mode=%s, method=%s'
% (batch_size, n_channel, str(img_shape), str(win_shape),
str(stride), border_mode, method))
x_shape = (batch_size, n_channel) + img_shape
x = np.random.normal(size=x_shape).astype(ca.float_)
layer = dp.Pool(win_shape=win_shape, method=method, strides=stride,
border_mode=border_mode)
layer._setup(x_shape)
y_img_shape = img_out_shape(img_shape, win_shape, stride, border_mode)
assert layer.y_shape(x_shape) == (batch_size, n_channel) + y_img_shape
check_grad(layer, x)
def run():
# Prepare data
dataset = dp.datasets.MNIST()
x, y = dataset.data()
x = x.astype(dp.float_)[:, np.newaxis, :, :]
y = y.astype(dp.int_)
train_idx, test_idx = dataset.split()
x_train = x[train_idx]
y_train = y[train_idx]
x_test = x[test_idx]
y_test = y[test_idx]
scaler = dp.UniformScaler(high=255.)
x_train = scaler.fit_transform(x_train)
x_test = scaler.transform(x_test)
batch_size = 128
train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size)
test_input = dp.SupervisedInput(x_test, y_test)
# Setup neural network
net = dp.NeuralNetwork(layers=[
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
weights=dp.Parameter(dp.AutoFiller(), weight_decay=0.0001),
),
dp.Activation('relu'),
dp.Pool(
win_shape=(3, 3),
strides=(2, 2),
method='max',
),
dp.Convolutional(
n_filters=64,
filter_shape=(5, 5),
weights=dp.Parameter(dp.AutoFiller(), weight_decay=0.0001),
),
dp.Activation('relu'),
dp.Pool(
win_shape=(3, 3),
strides=(2, 2),
method='max',
),
dp.Flatten(),
dp.FullyConnected(
n_output=128,
weights=dp.Parameter(dp.AutoFiller()),
),
dp.FullyConnected(
n_output=dataset.n_classes,
weights=dp.Parameter(dp.AutoFiller()),
),
dp.MultinomialLogReg(),
], )
# Train neural network
def val_error():
return net.error(test_input)
trainer = dp.StochasticGradientDescent(
max_epochs=15,
learn_rule=dp.Momentum(learn_rate=0.01, momentum=0.9),
)
trainer.train(net, train_input, val_error)
# Visualize convolutional filters to disk
for l, layer in enumerate(net.layers):
if not isinstance(layer, dp.Convolutional):
continue
W = np.array(layer.params()[0].array)
filepath = os.path.join('mnist', 'conv_layer_%i.png' % l)
dp.misc.img_save(dp.misc.conv_filter_tile(W), filepath)
# Evaluate on test data
error = net.error(test_input)
print('Test error rate: %.4f' % error)
'win_shape': (3, 3),
'strides': (2, 2),
'border_mode': 'same',
'method': 'max',
}
net = dp.NeuralNetwork(layers=[
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.0001),
weight_decay=0.004,
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
weight_decay=0.004,
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Convolutional(
n_filters=64,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
def run():
# Prepare data
batch_size = 128
dataset = dp.datasets.CIFAR10()
x, y = dataset.data()
y = y.astype(dp.int_)
train_idx, test_idx = dataset.split()
x_train = preprocess_imgs(x[train_idx])
y_train = y[train_idx]
train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size)
# Setup neural network
pool_kwargs = {
'win_shape': (3, 3),
'strides': (2, 2),
'border_mode': 'same',
'method': 'max',
}
nn = dp.NeuralNetwork(layers=[
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.0001),
penalty=('l2', 0.004),
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
penalty=('l2', 0.004),
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Convolutional(
n_filters=64,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
penalty=('l2', 0.004),
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Flatten(),
dp.FullyConnected(
n_output=64,
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
penalty=('l2', 0.03)),
),
dp.Activation('relu'),
dp.FullyConnected(
n_output=dataset.n_classes,
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
penalty=('l2', 0.03)),
),
dp.MultinomialLogReg(),
], )
dp.misc.profile(nn, train_input)
def run():
# Prepare data
batch_size = 128
dataset = dp.datasets.CIFAR10()
x, y = dataset.data()
y = y.astype(dp.int_)
train_idx, test_idx = dataset.split()
x_train = preprocess_imgs(x[train_idx])
y_train = y[train_idx]
x_test = preprocess_imgs(x[test_idx])
y_test = y[test_idx]
train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size)
test_input = dp.SupervisedInput(x_test, y_test, batch_size=batch_size)
# Setup neural network
pool_kwargs = {
'win_shape': (3, 3),
'strides': (2, 2),
'border_mode': 'same',
'method': 'max',
}
nn = dp.NeuralNetwork(layers=[
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.0001),
penalty=('l2', 0.004),
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
penalty=('l2', 0.004),
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Convolutional(
n_filters=64,
filter_shape=(5, 5),
border_mode='same',
weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
penalty=('l2', 0.004),
monitor=True),
),
dp.Activation('relu'),
dp.Pool(**pool_kwargs),
dp.Flatten(),
dp.FullyConnected(
n_output=64,
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
penalty=('l2', 0.03)),
),
dp.Activation('relu'),
dp.FullyConnected(
n_output=dataset.n_classes,
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
penalty=('l2', 0.03)),
),
dp.MultinomialLogReg(),
], )
# Train neural network
n_epochs = [8, 8]
learn_rate = 0.001
def valid_error():
return nn.error(test_input)
for i, max_epochs in enumerate(n_epochs):
lr = learn_rate / 10**i
trainer = dp.StochasticGradientDescent(
max_epochs=max_epochs,
learn_rule=dp.Momentum(learn_rate=lr, momentum=0.9),
)
trainer.train(nn, train_input, valid_error)
# Visualize convolutional filters to disk
for l, layer in enumerate(nn.layers):
if not isinstance(layer, dp.Convolutional):
continue
W = np.array(layer.params()[0].values)
dp.misc.img_save(dp.misc.conv_filter_tile(W),
os.path.join('cifar10', 'convnet_layer_%i.png' % l))
# Evaluate on test data
error = nn.error(test_input)
print('Test error rate: %.4f' % error)
def run():
# Prepare data
dataset = dp.datasets.MNIST()
x, y = dataset.data()
x = x[:, np.newaxis, :, :].astype(dp.float_) / 255.0 - 0.5
y = y.astype(dp.int_)
train_idx, test_idx = dataset.split()
x_train = x[train_idx]
y_train = y[train_idx]
x_test = x[test_idx]
y_test = y[test_idx]
train_input = dp.SupervisedInput(x_train, y_train, batch_size=128)
test_input = dp.SupervisedInput(x_test, y_test)
# Setup neural network
nn = dp.NeuralNetwork(layers=[
dp.Convolutional(
n_filters=20,
filter_shape=(5, 5),
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
penalty=('l2', 0.00001)),
),
dp.Activation('relu'),
dp.Pool(
win_shape=(2, 2),
strides=(2, 2),
method='max',
),
dp.Convolutional(
n_filters=50,
filter_shape=(5, 5),
weights=dp.Parameter(dp.NormalFiller(sigma=0.1),
penalty=('l2', 0.00001)),
),
dp.Activation('relu'),
dp.Pool(
win_shape=(2, 2),
strides=(2, 2),
method='max',
),
dp.Flatten(),
dp.FullyConnected(
n_output=500,
weights=dp.NormalFiller(sigma=0.01),
),
dp.FullyConnected(
n_output=dataset.n_classes,
weights=dp.NormalFiller(sigma=0.01),
),
dp.MultinomialLogReg(),
], )
# Train neural network
def valid_error():
return nn.error(test_input)
trainer = dp.StochasticGradientDescent(
max_epochs=15,
learn_rule=dp.Momentum(learn_rate=0.1, momentum=0.9),
)
trainer.train(nn, train_input, valid_error)
# Visualize convolutional filters to disk
for layer_idx, layer in enumerate(nn.layers):
if not isinstance(layer, dp.Convolutional):
continue
W = np.array(layer.params()[0].values)
dp.misc.img_save(
dp.misc.conv_filter_tile(W),
os.path.join('mnist', 'convnet_layer_%i.png' % layer_idx))
# Evaluate on test data
error = nn.error(test_input)
print('Test error rate: %.4f' % error)
import numpy as np
import deeppy as dp
net = dp.NeuralNetwork(layers=[
dp.Convolutional(
n_filters=32,
filter_shape=(5, 5),
weights=dp.Parameter(dp.AutoFiller(), weight_decay=0.0001),
),
dp.Activation('relu'),
dp.Pool(
win_shape=(3, 3),
strides=(2, 2),
method='max',
),
dp.Convolutional(
n_filters=64,
filter_shape=(5, 5),
weights=dp.Parameter(dp.AutoFiller(), weight_decay=0.0001),
),
dp.Activation('relu'),
dp.Pool(
win_shape=(3, 3),
strides=(2, 2),
method='max',
),
dp.Flatten(),
dp.FullyConnected(
n_output=128,
weights=dp.Parameter(dp.AutoFiller()),
),
