def pool_layer(pool_method, border_mode): return dp.Pool( win_shape=(3, 3), strides=(2, 2), method=pool_method, border_mode=border_mode, )
def pool_layer(): return dp.Pool( win_shape=(2, 2), strides=(2, 2), border_mode='valid', method='max', )
def run(): np.random.seed(3) layers = [ dp.Activation('relu'), dp.Activation('sigmoid'), dp.Activation('tanh'), dp.FullyConnected( n_output=3, weights=dp.NormalFiller(sigma=0.01), ), dp.Dropout(0.2), dp.DropoutFullyConnected( n_output=10, weights=dp.NormalFiller(sigma=0.01), dropout=0.5, ), ] input_shape = (1, 5) x = np.random.normal(size=input_shape).astype(dp.float_) for layer in layers: dp.misc.check_bprop(layer, x) conv_layers = [ dp.Convolutional( n_filters=32, filter_shape=(3, 3), border_mode='same', weights=dp.NormalFiller(sigma=0.01), ), dp.Convolutional( n_filters=32, filter_shape=(5, 5), border_mode='valid', weights=dp.NormalFiller(sigma=0.01), ), dp.Pool( win_shape=(3, 3), strides=(2, 2), method='max', ) ] input_shape = (5, 3, 8, 8) x = np.random.normal(size=input_shape).astype(dp.float_) for layer in conv_layers: dp.misc.check_bprop(layer, x)
def test_pool(): win_shapes = [(1, 1), (2, 2), (3, 3)] strides = [(1, 1), (2, 2), (3, 3)] border_modes = ['valid', 'same', 'full'] methods = ['max', 'avg'] confs = itertools.product(batch_sizes, n_channels, img_shapes, win_shapes, strides, border_modes, methods) # Sample random parameter configurations to reduce workload. confs = shuffled(confs)[:100] for (batch_size, n_channel, img_shape, win_shape, stride, border_mode, method) in confs: if img_shape[0] < win_shape[0] or img_shape[1] < win_shape[1]: continue if border_mode != 'valid' and \ (win_shape[0] != 1 or win_shape[1] != 1) and \ ca._backend == 'cuda' and \ ca.nnet.pool._default_impl == 'cudnn': # Bug: I think CUDArray/DeepPy calculates the padding in a manner # inconsistent with cuDNN continue if method == 'avg' and \ ca._backend == 'cuda' and \ ca.nnet.pool._default_impl == 'masked': # Not implemented yet continue print('Pool: batch_size=%i, n_channel=%i, img_shape=%s, win_shape=%s, ' 'stride=%s, border_mode=%s, method=%s' % (batch_size, n_channel, str(img_shape), str(win_shape), str(stride), border_mode, method)) x_shape = (batch_size, n_channel) + img_shape x = np.random.normal(size=x_shape).astype(ca.float_) layer = dp.Pool(win_shape=win_shape, method=method, strides=stride, border_mode=border_mode) layer._setup(x_shape) y_img_shape = img_out_shape(img_shape, win_shape, stride, border_mode) assert layer.y_shape(x_shape) == (batch_size, n_channel) + y_img_shape check_grad(layer, x)
def run(): # Prepare data dataset = dp.datasets.MNIST() x, y = dataset.data() x = x.astype(dp.float_)[:, np.newaxis, :, :] y = y.astype(dp.int_) train_idx, test_idx = dataset.split() x_train = x[train_idx] y_train = y[train_idx] x_test = x[test_idx] y_test = y[test_idx] scaler = dp.UniformScaler(high=255.) x_train = scaler.fit_transform(x_train) x_test = scaler.transform(x_test) batch_size = 128 train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size) test_input = dp.SupervisedInput(x_test, y_test) # Setup neural network net = dp.NeuralNetwork(layers=[ dp.Convolutional( n_filters=32, filter_shape=(5, 5), weights=dp.Parameter(dp.AutoFiller(), weight_decay=0.0001), ), dp.Activation('relu'), dp.Pool( win_shape=(3, 3), strides=(2, 2), method='max', ), dp.Convolutional( n_filters=64, filter_shape=(5, 5), weights=dp.Parameter(dp.AutoFiller(), weight_decay=0.0001), ), dp.Activation('relu'), dp.Pool( win_shape=(3, 3), strides=(2, 2), method='max', ), dp.Flatten(), dp.FullyConnected( n_output=128, weights=dp.Parameter(dp.AutoFiller()), ), dp.FullyConnected( n_output=dataset.n_classes, weights=dp.Parameter(dp.AutoFiller()), ), dp.MultinomialLogReg(), ], ) # Train neural network def val_error(): return net.error(test_input) trainer = dp.StochasticGradientDescent( max_epochs=15, learn_rule=dp.Momentum(learn_rate=0.01, momentum=0.9), ) trainer.train(net, train_input, val_error) # Visualize convolutional filters to disk for l, layer in enumerate(net.layers): if not isinstance(layer, dp.Convolutional): continue W = np.array(layer.params()[0].array) filepath = os.path.join('mnist', 'conv_layer_%i.png' % l) dp.misc.img_save(dp.misc.conv_filter_tile(W), filepath) # Evaluate on test data error = net.error(test_input) print('Test error rate: %.4f' % error)
'win_shape': (3, 3), 'strides': (2, 2), 'border_mode': 'same', 'method': 'max', } net = dp.NeuralNetwork(layers=[ dp.Convolutional( n_filters=32, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.0001), weight_decay=0.004, monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs), dp.Convolutional( n_filters=32, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.01), weight_decay=0.004, monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs), dp.Convolutional( n_filters=64, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.01),
def run(): # Prepare data batch_size = 128 dataset = dp.datasets.CIFAR10() x, y = dataset.data() y = y.astype(dp.int_) train_idx, test_idx = dataset.split() x_train = preprocess_imgs(x[train_idx]) y_train = y[train_idx] train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size) # Setup neural network pool_kwargs = { 'win_shape': (3, 3), 'strides': (2, 2), 'border_mode': 'same', 'method': 'max', } nn = dp.NeuralNetwork(layers=[ dp.Convolutional( n_filters=32, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.0001), penalty=('l2', 0.004), monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs), dp.Convolutional( n_filters=32, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.01), penalty=('l2', 0.004), monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs), dp.Convolutional( n_filters=64, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.01), penalty=('l2', 0.004), monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs), dp.Flatten(), dp.FullyConnected( n_output=64, weights=dp.Parameter(dp.NormalFiller(sigma=0.1), penalty=('l2', 0.03)), ), dp.Activation('relu'), dp.FullyConnected( n_output=dataset.n_classes, weights=dp.Parameter(dp.NormalFiller(sigma=0.1), penalty=('l2', 0.03)), ), dp.MultinomialLogReg(), ], ) dp.misc.profile(nn, train_input)
def run(): # Prepare data batch_size = 128 dataset = dp.datasets.CIFAR10() x, y = dataset.data() y = y.astype(dp.int_) train_idx, test_idx = dataset.split() x_train = preprocess_imgs(x[train_idx]) y_train = y[train_idx] x_test = preprocess_imgs(x[test_idx]) y_test = y[test_idx] train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size) test_input = dp.SupervisedInput(x_test, y_test, batch_size=batch_size) # Setup neural network pool_kwargs = { 'win_shape': (3, 3), 'strides': (2, 2), 'border_mode': 'same', 'method': 'max', } nn = dp.NeuralNetwork(layers=[ dp.Convolutional( n_filters=32, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.0001), penalty=('l2', 0.004), monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs), dp.Convolutional( n_filters=32, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.01), penalty=('l2', 0.004), monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs), dp.Convolutional( n_filters=64, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.01), penalty=('l2', 0.004), monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs), dp.Flatten(), dp.FullyConnected( n_output=64, weights=dp.Parameter(dp.NormalFiller(sigma=0.1), penalty=('l2', 0.03)), ), dp.Activation('relu'), dp.FullyConnected( n_output=dataset.n_classes, weights=dp.Parameter(dp.NormalFiller(sigma=0.1), penalty=('l2', 0.03)), ), dp.MultinomialLogReg(), ], ) # Train neural network n_epochs = [8, 8] learn_rate = 0.001 def valid_error(): return nn.error(test_input) for i, max_epochs in enumerate(n_epochs): lr = learn_rate / 10**i trainer = dp.StochasticGradientDescent( max_epochs=max_epochs, learn_rule=dp.Momentum(learn_rate=lr, momentum=0.9), ) trainer.train(nn, train_input, valid_error) # Visualize convolutional filters to disk for l, layer in enumerate(nn.layers): if not isinstance(layer, dp.Convolutional): continue W = np.array(layer.params()[0].values) dp.misc.img_save(dp.misc.conv_filter_tile(W), os.path.join('cifar10', 'convnet_layer_%i.png' % l)) # Evaluate on test data error = nn.error(test_input) print('Test error rate: %.4f' % error)
def run(): # Prepare data dataset = dp.datasets.MNIST() x, y = dataset.data() x = x[:, np.newaxis, :, :].astype(dp.float_) / 255.0 - 0.5 y = y.astype(dp.int_) train_idx, test_idx = dataset.split() x_train = x[train_idx] y_train = y[train_idx] x_test = x[test_idx] y_test = y[test_idx] train_input = dp.SupervisedInput(x_train, y_train, batch_size=128) test_input = dp.SupervisedInput(x_test, y_test) # Setup neural network nn = dp.NeuralNetwork(layers=[ dp.Convolutional( n_filters=20, filter_shape=(5, 5), weights=dp.Parameter(dp.NormalFiller(sigma=0.1), penalty=('l2', 0.00001)), ), dp.Activation('relu'), dp.Pool( win_shape=(2, 2), strides=(2, 2), method='max', ), dp.Convolutional( n_filters=50, filter_shape=(5, 5), weights=dp.Parameter(dp.NormalFiller(sigma=0.1), penalty=('l2', 0.00001)), ), dp.Activation('relu'), dp.Pool( win_shape=(2, 2), strides=(2, 2), method='max', ), dp.Flatten(), dp.FullyConnected( n_output=500, weights=dp.NormalFiller(sigma=0.01), ), dp.FullyConnected( n_output=dataset.n_classes, weights=dp.NormalFiller(sigma=0.01), ), dp.MultinomialLogReg(), ], ) # Train neural network def valid_error(): return nn.error(test_input) trainer = dp.StochasticGradientDescent( max_epochs=15, learn_rule=dp.Momentum(learn_rate=0.1, momentum=0.9), ) trainer.train(nn, train_input, valid_error) # Visualize convolutional filters to disk for layer_idx, layer in enumerate(nn.layers): if not isinstance(layer, dp.Convolutional): continue W = np.array(layer.params()[0].values) dp.misc.img_save( dp.misc.conv_filter_tile(W), os.path.join('mnist', 'convnet_layer_%i.png' % layer_idx)) # Evaluate on test data error = nn.error(test_input) print('Test error rate: %.4f' % error)
import numpy as np import deeppy as dp net = dp.NeuralNetwork(layers=[ dp.Convolutional( n_filters=32, filter_shape=(5, 5), weights=dp.Parameter(dp.AutoFiller(), weight_decay=0.0001), ), dp.Activation('relu'), dp.Pool( win_shape=(3, 3), strides=(2, 2), method='max', ), dp.Convolutional( n_filters=64, filter_shape=(5, 5), weights=dp.Parameter(dp.AutoFiller(), weight_decay=0.0001), ), dp.Activation('relu'), dp.Pool( win_shape=(3, 3), strides=(2, 2), method='max', ), dp.Flatten(), dp.FullyConnected( n_output=128, weights=dp.Parameter(dp.AutoFiller()), ),