def test_classification(): # Make dataset n_classes = 2 n_samples = 1000 n_features = 48 x, y = make_classification(n_samples=n_samples, n_features=n_features, n_classes=n_classes, n_informative=n_classes * 2, random_state=1) x = x.astype(dp.float_) y = y.astype(dp.int_) n_train = int(0.8 * n_samples) x_train = x[:n_train] y_train = y[:n_train] x_test = x[n_train:] y_test = y[n_train:] scaler = dp.StandardScaler() x_train = scaler.fit_transform(x_train) x_test = scaler.transform(x_test) # Setup feeds batch_size = 16 train_feed = dp.SupervisedFeed(x_train, y_train, batch_size=batch_size) test_feed = dp.Feed(x_test) # Setup neural network weight_decay = 1e-03 net = dp.NeuralNetwork( layers=[ dp.Affine( n_out=32, weights=dp.Parameter(dp.AutoFiller(), weight_decay=weight_decay), ), dp.ReLU(), dp.Affine( n_out=64, weights=dp.Parameter(dp.AutoFiller(), weight_decay=weight_decay), ), dp.ReLU(), dp.Affine( n_out=n_classes, weights=dp.Parameter(dp.AutoFiller()), ), ], loss=dp.SoftmaxCrossEntropy(), ) # Train neural network learn_rule = dp.Momentum(learn_rate=0.01 / batch_size, momentum=0.9) trainer = dp.GradientDescent(net, train_feed, learn_rule) trainer.train_epochs(n_epochs=10) # Evaluate on test data error = np.mean(net.predict(test_feed) != y_test) print('Test error rate: %.4f' % error) assert error < 0.2
def test_fully_connected(): confs = itertools.product(batch_sizes, n_ins, n_outs) for batch_size, n_in, n_out in confs: print('Affine: batch_size=%i, n_in=%i, n_out=%i' % (batch_size, n_in, n_out)) x_shape = (batch_size, n_in) x = np.random.normal(size=x_shape).astype(ca.float_) w = np.random.normal(size=(n_in, n_out)).astype(ca.float_) b = np.random.normal(size=n_out).astype(ca.float_) layer = dp.Affine(n_out, weights=w, bias=b) layer.setup(x_shape) assert layer.y_shape(x_shape) == (batch_size, n_out) y = np.array(layer.fprop(ca.array(x))) assert allclose(np.dot(x, w) + b, y) y_grad = y x_grad = np.array(layer.bprop(ca.array(y_grad))) assert allclose(np.dot(y, w.T), x_grad) check_grad(layer, x) check_params(layer)
y[n] = 1 else: y[n] = 0 n += 1 # Prepare network feeds batch_size = 128 train_feed = dp.SupervisedSiameseFeed(x1, x2, y, batch_size=batch_size) # Setup network w_gain = 1.5 w_decay = 1e-4 net = dp.SiameseNetwork( siamese_layers=[ dp.Affine( n_out=1024, weights=dp.Parameter(dp.AutoFiller(w_gain), weight_decay=w_decay), ), dp.ReLU(), dp.Affine( n_out=1024, weights=dp.Parameter(dp.AutoFiller(w_gain), weight_decay=w_decay), ), dp.ReLU(), dp.Affine( n_out=2, weights=dp.Parameter(dp.AutoFiller(w_gain)), ), ], loss=dp.ContrastiveLoss(margin=1.0), )
def test_classification(): # Make dataset n_classes = 2 n_samples = 1000 n_features = 48 x, y = make_classification( n_samples=n_samples, n_features=n_features, n_classes=n_classes, n_informative=n_classes*2, random_state=1 ) n_train = int(0.8 * n_samples) n_val = int(0.5 * (n_samples - n_train)) x_train = x[:n_train] y_train = y[:n_train] x_val = x[n_train:n_train+n_val] y_val = y[n_train:n_train+n_val] x_test = x[n_train+n_val:] y_test = y[n_train+n_val:] scaler = dp.StandardScaler() x_train = scaler.fit_transform(x_train) x_val = scaler.transform(x_val) x_test = scaler.transform(x_test) # Setup input batch_size = 16 train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size) val_input = dp.Input(x_val) test_input = dp.Input(x_test) # Setup neural network weight_decay = 1e-03 net = dp.NeuralNetwork( layers=[ dp.Affine( n_out=32, weights=dp.Parameter(dp.AutoFiller(), weight_decay=weight_decay), ), dp.ReLU(), dp.Affine( n_out=64, weights=dp.Parameter(dp.AutoFiller(), weight_decay=weight_decay), ), dp.ReLU(), dp.Affine( n_out=n_classes, weights=dp.Parameter(dp.AutoFiller()), ), ], loss=dp.SoftmaxCrossEntropy(), ) # Train neural network def val_error(): return np.mean(net.predict(val_input) != y_val) trainer = dp.GradientDescent( min_epochs=10, learn_rule=dp.Momentum(learn_rate=0.01, momentum=0.9), ) trainer.train(net, train_input, val_error) # Evaluate on test data error = np.mean(net.predict(test_input) != y_test) print('Test error rate: %.4f' % error) assert error < 0.2
method='max', ) net = dp.NeuralNetwork( layers=[ conv_layer(32), dp.ReLU(), pool_layer(), conv_layer(32), dp.ReLU(), pool_layer(), conv_layer(64), dp.ReLU(), pool_layer(), dp.Flatten(), dp.Dropout(), dp.Affine(n_out=64, weights=dp.Parameter(dp.AutoFiller(gain=1.25), weight_decay=0.03)), dp.ReLU(), dp.Affine( n_out=dataset.n_classes, weights=dp.Parameter(dp.AutoFiller(gain=1.25)), ) ], loss=dp.SoftmaxCrossEntropy(), ) profile(net, train_input)
n_epochs = 25 learn_rate = 0.025 / batch_size learn_rule = dp.Momentum(learn_rate, momentum=0.9) for ae in sae.ae_models(): trainer = dp.GradientDescent(ae, train_feed, learn_rule) trainer.train_epochs(n_epochs) # Train stacked autoencoders trainer = dp.GradientDescent(sae, train_feed, learn_rule) trainer.train_epochs(n_epochs) # Setup neural network using the stacked autoencoder layers net = dp.NeuralNetwork( layers=sae.feedforward_layers() + [ dp.Affine( n_out=dataset.n_classes, weights=dp.Parameter(dp.AutoFiller()), ), ], loss=dp.SoftmaxCrossEntropy(), ) # Fine-tune neural network train_feed = dp.SupervisedFeed(x_train, y_train, batch_size=batch_size) test_feed = dp.Feed(x_test) trainer = dp.GradientDescent(net, train_feed, learn_rule) trainer.train_epochs(n_epochs) # Evaluate on test data error = np.mean(net.predict(test_feed) != y_test) print('Test error rate: %.4f' % error)
x_train = scaler.fit_transform(x_train) x_test = scaler.transform(x_test) # Prepare network feeds batch_size = 128 train_feed = dp.SupervisedFeed(x_train, y_train, batch_size=batch_size) test_feed = dp.Feed(x_test) # Setup network weight_gain = 2.0 weight_decay = 0.0005 net = dp.NeuralNetwork( layers=[ dp.Affine( n_out=1024, weights=dp.Parameter(dp.AutoFiller(weight_gain), weight_decay=weight_decay), ), dp.ReLU(), dp.Affine( n_out=1024, weights=dp.Parameter(dp.AutoFiller(weight_gain), weight_decay=weight_decay), ), dp.ReLU(), dp.Affine( n_out=dataset.n_classes, weights=dp.Parameter(dp.AutoFiller()), ), ], loss=dp.SoftmaxCrossEntropy(),
weight_gain_fc = 1.84 weight_decay_fc = 0.002 net = dp.NeuralNetwork( layers=[ conv_layer(32), dp.ReLU(), pool_layer(), conv_layer(64), dp.ReLU(), pool_layer(), dp.Flatten(), dp.Dropout(), dp.Affine( n_out=512, weights=dp.Parameter(dp.AutoFiller(weight_gain_fc), weight_decay=weight_decay_fc), ), dp.ReLU(), dp.Affine( n_out=dataset.n_classes, weights=dp.Parameter(dp.AutoFiller(weight_gain_fc)), ), ], loss=dp.SoftmaxCrossEntropy(), ) # Train network n_epochs = [50, 15, 15] learn_rate = 0.05 / batch_size learn_rule = dp.Momentum(momentum=0.9)