def run(): # Prepare data dataset = dp.datasets.MNIST() x, y = dataset.data(flat=True) x = x.astype(dp.float_)/255.0 y = y.astype(dp.int_) train_idx, test_idx = dataset.split() x_train = x[train_idx] y_train = y[train_idx] x_test = x[test_idx] y_test = y[test_idx] train_input = dp.SupervisedInput(x_train, y_train, batch_size=128) test_input = dp.SupervisedInput(x_test, y_test) # Setup neural network nn = dp.NeuralNetwork( layers=[ dp.Dropout(0.2), dp.DropoutFullyConnected( n_output=800, dropout=0.5, weights=dp.Parameter(dp.NormalFiller(sigma=0.01), penalty=('l2', 0.00001), monitor=True), ), dp.Activation('relu'), dp.DropoutFullyConnected( n_output=800, dropout=0.5, weights=dp.Parameter(dp.NormalFiller(sigma=0.01), penalty=('l2', 0.00001), monitor=True), ), dp.Activation('relu'), dp.DropoutFullyConnected( n_output=dataset.n_classes, weights=dp.Parameter(dp.NormalFiller(sigma=0.01), penalty=('l2', 0.00001), monitor=True), ), dp.MultinomialLogReg(), ], ) # Train neural network def valid_error(): return nn.error(test_input) trainer = dp.StochasticGradientDescent( max_epochs=50, learn_rule=dp.Momentum(learn_rate=0.1, momentum=0.9), ) trainer.train(nn, train_input, valid_error) # Visualize weights from first layer W = next(np.array(layer.params()[0].values) for layer in nn.layers if isinstance(layer, dp.FullyConnected)) W = np.reshape(W.T, (-1, 28, 28)) dp.misc.img_save(dp.misc.img_tile(dp.misc.img_stretch(W)), os.path.join('mnist', 'mlp_dropout_weights.png')) # Evaluate on test data error = nn.error(test_input) print('Test error rate: %.4f' % error)
def run(): np.random.seed(3) layers = [ dp.Activation('relu'), dp.Activation('sigmoid'), dp.Activation('tanh'), dp.FullyConnected( n_output=3, weights=dp.NormalFiller(sigma=0.01), ), dp.Dropout(0.2), dp.DropoutFullyConnected( n_output=10, weights=dp.NormalFiller(sigma=0.01), dropout=0.5, ), ] input_shape = (1, 5) x = np.random.normal(size=input_shape).astype(dp.float_) for layer in layers: dp.misc.check_bprop(layer, x) conv_layers = [ dp.Convolutional( n_filters=32, filter_shape=(3, 3), border_mode='same', weights=dp.NormalFiller(sigma=0.01), ), dp.Convolutional( n_filters=32, filter_shape=(5, 5), border_mode='valid', weights=dp.NormalFiller(sigma=0.01), ), dp.Pool( win_shape=(3, 3), strides=(2, 2), method='max', ) ] input_shape = (5, 3, 8, 8) x = np.random.normal(size=input_shape).astype(dp.float_) for layer in conv_layers: dp.misc.check_bprop(layer, x)
def __init__(self, game_name, run_id): self.number_of_actions = len(action_dict[game_name]) valid_actions = action_dict[game_name] net.layers[-2] = dp.FullyConnected(n_output=self.number_of_actions, weights=dp.Parameter( dp.NormalFiller(sigma=0.1), weight_decay=0.004, monitor=False)) self.memory = MemoryD(self.memory_size) self.ale = ALE(valid_actions, run_id, display_screen="false", skip_frames=4, game_ROM='ale/roms/' + game_name + '.bin') self.nnet = net self.q_values = [] self.test_game_scores = []
import numpy as np import deeppy as dp # Setup neural network pool_kwargs = { 'win_shape': (3, 3), 'strides': (2, 2), 'border_mode': 'same', 'method': 'max', } net = dp.NeuralNetwork(layers=[ dp.Convolutional( n_filters=32, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.0001), weight_decay=0.004, monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs), dp.Convolutional( n_filters=32, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.01), weight_decay=0.004, monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs),
def run(): # Prepare data batch_size = 128 dataset = dp.datasets.CIFAR10() x, y = dataset.data() y = y.astype(dp.int_) train_idx, test_idx = dataset.split() x_train = preprocess_imgs(x[train_idx]) y_train = y[train_idx] train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size) # Setup neural network pool_kwargs = { 'win_shape': (3, 3), 'strides': (2, 2), 'border_mode': 'same', 'method': 'max', } nn = dp.NeuralNetwork(layers=[ dp.Convolutional( n_filters=32, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.0001), penalty=('l2', 0.004), monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs), dp.Convolutional( n_filters=32, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.01), penalty=('l2', 0.004), monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs), dp.Convolutional( n_filters=64, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.01), penalty=('l2', 0.004), monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs), dp.Flatten(), dp.FullyConnected( n_output=64, weights=dp.Parameter(dp.NormalFiller(sigma=0.1), penalty=('l2', 0.03)), ), dp.Activation('relu'), dp.FullyConnected( n_output=dataset.n_classes, weights=dp.Parameter(dp.NormalFiller(sigma=0.1), penalty=('l2', 0.03)), ), dp.MultinomialLogReg(), ], ) dp.misc.profile(nn, train_input)
def run(): # Prepare data batch_size = 128 dataset = dp.datasets.CIFAR10() x, y = dataset.data() y = y.astype(dp.int_) train_idx, test_idx = dataset.split() x_train = preprocess_imgs(x[train_idx]) y_train = y[train_idx] x_test = preprocess_imgs(x[test_idx]) y_test = y[test_idx] train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size) test_input = dp.SupervisedInput(x_test, y_test, batch_size=batch_size) # Setup neural network pool_kwargs = { 'win_shape': (3, 3), 'strides': (2, 2), 'border_mode': 'same', 'method': 'max', } nn = dp.NeuralNetwork(layers=[ dp.Convolutional( n_filters=32, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.0001), penalty=('l2', 0.004), monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs), dp.Convolutional( n_filters=32, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.01), penalty=('l2', 0.004), monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs), dp.Convolutional( n_filters=64, filter_shape=(5, 5), border_mode='same', weights=dp.Parameter(dp.NormalFiller(sigma=0.01), penalty=('l2', 0.004), monitor=True), ), dp.Activation('relu'), dp.Pool(**pool_kwargs), dp.Flatten(), dp.FullyConnected( n_output=64, weights=dp.Parameter(dp.NormalFiller(sigma=0.1), penalty=('l2', 0.03)), ), dp.Activation('relu'), dp.FullyConnected( n_output=dataset.n_classes, weights=dp.Parameter(dp.NormalFiller(sigma=0.1), penalty=('l2', 0.03)), ), dp.MultinomialLogReg(), ], ) # Train neural network n_epochs = [8, 8] learn_rate = 0.001 def valid_error(): return nn.error(test_input) for i, max_epochs in enumerate(n_epochs): lr = learn_rate / 10**i trainer = dp.StochasticGradientDescent( max_epochs=max_epochs, learn_rule=dp.Momentum(learn_rate=lr, momentum=0.9), ) trainer.train(nn, train_input, valid_error) # Visualize convolutional filters to disk for l, layer in enumerate(nn.layers): if not isinstance(layer, dp.Convolutional): continue W = np.array(layer.params()[0].values) dp.misc.img_save(dp.misc.conv_filter_tile(W), os.path.join('cifar10', 'convnet_layer_%i.png' % l)) # Evaluate on test data error = nn.error(test_input) print('Test error rate: %.4f' % error)
def run(): # Prepare data dataset = dp.datasets.MNIST() x, y = dataset.data() x = x[:, np.newaxis, :, :].astype(dp.float_) / 255.0 - 0.5 y = y.astype(dp.int_) train_idx, test_idx = dataset.split() x_train = x[train_idx] y_train = y[train_idx] x_test = x[test_idx] y_test = y[test_idx] train_input = dp.SupervisedInput(x_train, y_train, batch_size=128) test_input = dp.SupervisedInput(x_test, y_test) # Setup neural network nn = dp.NeuralNetwork(layers=[ dp.Convolutional( n_filters=20, filter_shape=(5, 5), weights=dp.Parameter(dp.NormalFiller(sigma=0.1), penalty=('l2', 0.00001)), ), dp.Activation('relu'), dp.Pool( win_shape=(2, 2), strides=(2, 2), method='max', ), dp.Convolutional( n_filters=50, filter_shape=(5, 5), weights=dp.Parameter(dp.NormalFiller(sigma=0.1), penalty=('l2', 0.00001)), ), dp.Activation('relu'), dp.Pool( win_shape=(2, 2), strides=(2, 2), method='max', ), dp.Flatten(), dp.FullyConnected( n_output=500, weights=dp.NormalFiller(sigma=0.01), ), dp.FullyConnected( n_output=dataset.n_classes, weights=dp.NormalFiller(sigma=0.01), ), dp.MultinomialLogReg(), ], ) # Train neural network def valid_error(): return nn.error(test_input) trainer = dp.StochasticGradientDescent( max_epochs=15, learn_rule=dp.Momentum(learn_rate=0.1, momentum=0.9), ) trainer.train(nn, train_input, valid_error) # Visualize convolutional filters to disk for layer_idx, layer in enumerate(nn.layers): if not isinstance(layer, dp.Convolutional): continue W = np.array(layer.params()[0].values) dp.misc.img_save( dp.misc.conv_filter_tile(W), os.path.join('mnist', 'convnet_layer_%i.png' % layer_idx)) # Evaluate on test data error = nn.error(test_input) print('Test error rate: %.4f' % error)
def run(): # Fetch MNIST data dataset = dp.datasets.MNIST() x, y = dataset.data(flat=True) x = x.astype(dp.float_) / 255.0 y = y.astype(dp.int_) train_idx, test_idx = dataset.split() x_train = x[train_idx] y_train = y[train_idx] x_test = x[test_idx] y_test = y[test_idx] # Generate image pairs n_pairs = 100000 x1 = np.empty((n_pairs, 28 * 28), dtype=dp.float_) x2 = np.empty_like(x1, dtype=dp.float_) y = np.empty(n_pairs, dtype=dp.int_) n_imgs = x_train.shape[0] n = 0 while n < n_pairs: i = random.randint(0, n_imgs - 1) j = random.randint(0, n_imgs - 1) if i == j: continue x1[n, ...] = x_train[i] x2[n, ...] = x_train[j] if y_train[i] == y_train[j]: y[n] = 1 else: y[n] = 0 n += 1 # Input to network train_input = dp.SupervisedSiameseInput(x1, x2, y, batch_size=128) test_input = dp.SupervisedInput(x_test, y_test) # Setup network net = dp.SiameseNetwork( siamese_layers=[ dp.Dropout(), dp.FullyConnected( n_output=800, weights=dp.Parameter(dp.NormalFiller(sigma=0.02), penalty=('l2', 0.00001), monitor=True), ), dp.Activation('relu'), dp.FullyConnected( n_output=800, weights=dp.Parameter(dp.NormalFiller(sigma=0.02), penalty=('l2', 0.00001), monitor=True), ), dp.Activation('relu'), dp.FullyConnected( n_output=2, weights=dp.Parameter(dp.NormalFiller(sigma=0.1), penalty=('l2', 0.00001), monitor=True), ), ], loss_layer=dp.ContrastiveLoss(), ) # Train network trainer = dp.StochasticGradientDescent( max_epochs=5, learn_rule=dp.Momentum(learn_rate=0.1, momentum=0.9), ) trainer.train(net, train_input) # Visualize feature space feat = net.features(test_input) colors = [ 'tomato', 'lawngreen', 'royalblue', 'gold', 'saddlebrown', 'violet', 'turquoise', 'mediumpurple', 'darkorange', 'darkgray' ] plt.figure() for i in range(10): plt.scatter(feat[y_test == i, 0], feat[y_test == i, 1], s=3, c=colors[i], linewidths=0) plt.legend([str(i) for i in range(10)], scatterpoints=1, markerscale=4) plt.savefig(os.path.join('mnist', 'siamese_dists.png'), dpi=200)