def test_cnn(dataset: str):
if dataset == 'mnist':
mnist = fetch_mldata("MNIST original")
X = mnist.data / 255
y = np.reshape(mnist.target, (mnist.target.shape[0]))
X = X.reshape((X.shape[0], 28, 28, 1))
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)
#X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=512)
batch_size = 512
epochs = 10
optimizer = AdamOptimizer(learning_rate=.005)
nn = NeuralStack()
nn.add(Convolutional2D([5, 5, 1], 32, strides=(2, 2)))
nn.add(Activation('relu'))
nn.add(Convolutional2D([5, 5, 32], 32, strides=(2, 2)))
nn.add(Activation('relu'))
nn.add(Flatten())
nn.add(FullyConnected(512, 64))
nn.add(Activation('sigmoid'))
nn.add(FullyConnected(64, 10))
nn.add(Activation('softmax'))
elif dataset == 'digits':
digits = load_digits()
X = digits.images
y = digits.target
X = X.reshape((X.shape[0], 8, 8, 1))
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)
optimizer = AdamOptimizer(learning_rate=.001)
nn = NeuralStack()
nn.add(Convolutional2D([4, 4, 1], 32, strides=(2, 2)))
nn.add(Activation('relu'))
#nn.add(Convolutional2D([3, 3, 32], 32, strides=(1, 1)))
#nn.add(Activation('relu'))
#nn.add(Convolutional2D([2, 2, 32], 64, strides=(1, 1)))
#nn.add(Activation('relu'))
nn.add(Flatten())
nn.add(FullyConnected(288, 200))
nn.add(Activation('relu'))
nn.add(FullyConnected(200, 10))
nn.add(Activation('softmax'))
batch_size = 1024
epochs = 50
nn.add(optimizer)
nn.add(CrossEntropy())
nn.add(SGDBatching(batch_size))
errors = nn.train(X_train, y_train,
epochs=epochs, print_every=1*batch_size, validation_fraction=0.01, alpha_reg=.005)
accuracy_test = nn.get_accuracy(X_test, y_test, batch_size=1024)
log('Accuracy on test: {:.3f}%'.format(accuracy_test))
plt.plot(errors)
plt.show()
def test_cnn_digits():
images, labels = load_digits(return_X_y=True)
images = images.reshape((-1, 8, 8))
X_train, X_test, y_train, y_test = train_test_split(images,
labels,
train_size=.9,
test_size=.1)
nn = NeuralStack()
nn.add(Convolutional2D((4, 4, 1), 16))
nn.add(Activation('relu'))
nn.add(MaxPooling2D((2, 2)))
nn.add(Flatten())
nn.add(FullyConnected(256, 64))
nn.add(Activation('sigmoid'))
nn.add(FullyConnected(64, 10))
nn.add(Activation('softmax'))
nn.add(SGDBatching(len(X_train)))
nn.add(CrossEntropy())
nn.add(AdamOptimizer(learning_rate=5e-3))
nn.train(X_train,
y_train,
print_every=5,
epochs=150,
l2_alpha=.05,
validation_fraction=0)
print('Trained')
assert ClassificationCost.accuracy(y_test, nn.eval(X_test)) >= .9
def train_mlp(use_lbfgs=True):
mnist = fetch_mldata("MNIST original")
X = (mnist.data / 255 - .5) * 2
y = np.reshape(mnist.target, (mnist.target.shape[0], 1))
X = X.reshape((X.shape[0], 28, 28, 1)) # New shape: (None, 28, 28, 1)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=1. / 7.)
nn = NeuralStack()
n_hidden = 500
if use_lbfgs:
nn.add(Flatten())
nn.add(FullyConnected(28 * 28, n_hidden))
nn.add(Activation(function='sigmoid'))
nn.add(FullyConnected(n_hidden, 10))
nn.add(Activation(function='softmax'))
nn.add(CrossEntropy())
nn.add(SGDBatching(256))
adam = AdamOptimizer(learning_rate=0.007)
nn.add(LBFGS(m=30, first_order_optimizer=adam))
else:
nn.add(Flatten())
nn.add(FullyConnected(28 * 28, n_hidden))
nn.add(Activation(function='sigmoid'))
nn.add(FullyConnected(n_hidden, 10))
nn.add(Activation(function='softmax'))
nn.add(CrossEntropy())
nn.add(SGDBatching(512))
nn.add(AdamOptimizer(learning_rate=0.007))
t0 = time()
nn.train(X_train, y_train, alpha_reg=0.0001, epochs=6, print_every=100)
train_acc = accuracy_score(np.squeeze(y_train),
nn.eval(X_train).argmax(axis=1))
test_acc = accuracy_score(np.squeeze(y_test),
nn.eval(X_test).argmax(axis=1))
print("Training accuracy: %f" % train_acc)
print("Test accuracy: %f" % test_acc)
print("Optimization time: %.2f" % (time() - t0))
def get_trained_autoencoder():
try:
with open('autoencoder.pickle', 'rb') as f:
return pickle.load(f)
except FileNotFoundError:
nn = create_autoencoder()
nn.add(SGDBatching(2048))
nn.train(X_train, X_train, l2_alpha=.1, epochs=50, print_every=5)
nn.batch_op = None
with open('autoencoder.pickle', 'wb') as f:
pickle.dump(nn, f)
return nn
def test_mlp_on_xor_problem():
nn = NeuralStack()
nn.add(FullyConnected(2, 8))
nn.add(Activation(function='tanh'))
nn.add(FullyConnected(8, 1))
nn.add(Activation(function='sigmoid'))
nn.add(MSE())
nn.add(MomentumOptimizer(learning_rate=0.5, momentum=0))
nn.add(SGDBatching(4))
X = [[0, 0], [0, 1], [1, 0], [1, 1]]
y = [0, 1, 1, 0]
nn.train(X, y, alpha_reg=0, epochs=1000, print_every=800)
predictions = np.squeeze(nn.eval(X))
assert np.array_equal(predictions > 0.5, y)
def test_mlp_digits():
images, labels = load_digits(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(images,
labels,
train_size=.9,
test_size=.1)
nn = NeuralStack()
nn.add(FullyConnected(64, 32))
nn.add(Activation('relu'))
nn.add(FullyConnected(32, 10))
nn.add(Activation('softmax'))
nn.add(SGDBatching(len(X_train)))
nn.add(CrossEntropy())
nn.add(AdamOptimizer(learning_rate=5e-3))
nn.train(X_train, y_train, print_every=1, epochs=100, l2_alpha=.05)
assert ClassificationCost.accuracy(y_test, nn.eval(X_test)) >= .9