def run():
# Fetch data
mnist = sklearn.datasets.fetch_mldata('MNIST original', data_home='./data')
split = 60000
X_train = np.reshape(mnist.data[:split], (-1, 1, 28, 28)) / 255.0
y_train = mnist.target[:split]
X_test = np.reshape(mnist.data[split:], (-1, 1, 28, 28)) / 255.0
y_test = mnist.target[split:]
n_classes = np.unique(y_train).size
# Downsample training data
n_train_samples = 3000
train_idxs = np.random.random_integers(0, split - 1, n_train_samples)
X_train = X_train[train_idxs, ...]
y_train = y_train[train_idxs, ...]
# Setup convolutional neural network
nn = nnet.NeuralNetwork(layers=[
nnet.Conv(
n_feats=12,
filter_shape=(5, 5),
strides=(1, 1),
weight_scale=0.1,
weight_decay=0.001,
),
nnet.Activation('relu'),
nnet.Pool(
pool_shape=(2, 2),
strides=(2, 2),
mode='max',
),
nnet.Conv(
n_feats=16,
filter_shape=(5, 5),
strides=(1, 1),
weight_scale=0.1,
weight_decay=0.001,
),
nnet.Activation('relu'),
nnet.Flatten(),
nnet.Linear(
n_out=n_classes,
weight_scale=0.1,
weight_decay=0.02,
),
nnet.LogRegression(),
], )
# Train neural network
t0 = time.time()
nn.fit(X_train, y_train, learning_rate=0.05, max_iter=3, batch_size=32)
t1 = time.time()
print('Duration: %.1fs' % (t1 - t0))
# Evaluate on test data
error = nn.error(X_test, y_test)
print('Test error rate: %.4f' % error)
def run():
# Fetch data
digits = sklearn.datasets.load_digits()
X_train = digits.data
X_train /= np.max(X_train)
y_train = digits.target
n_classes = np.unique(y_train).size
# Setup multi-layer perceptron
nn = nnet.NeuralNetwork(layers=[
nnet.Linear(
n_out=50,
weight_scale=0.1,
weight_decay=0.002,
),
nnet.Activation('relu'),
nnet.Linear(
n_out=n_classes,
weight_scale=0.1,
weight_decay=0.002,
),
nnet.LogRegression(),
], )
# Verify network for correct back-propagation of parameter gradients
print('Checking gradients')
nn.check_gradients(X_train[:100], y_train[:100])
# Train neural network
print('Training neural network')
nn.fit(X_train, y_train, learning_rate=0.1, max_iter=25, batch_size=32)
# Evaluate on training data
error = nn.error(X_train, y_train)
print('Training error rate: %.4f' % error)
def run():
# Fetch data
mnist = sklearn.datasets.fetch_mldata('MNIST original', data_home='./data')
split = 60000
X_train = mnist.data[:split] / 255.0
y_train = mnist.target[:split]
X_test = mnist.data[split:] / 255.0
y_test = mnist.target[split:]
n_classes = np.unique(y_train).size
# Downsample training data
n_train_samples = 10000
train_idxs = np.random.random_integers(0, split - 1, n_train_samples)
X_train = X_train[train_idxs, ...]
y_train = y_train[train_idxs, ...]
# Setup multi-layer perceptron
nn = nnet.NeuralNetwork(layers=[
nnet.Linear(
n_out=100,
weight_scale=0.2,
weight_decay=0.004,
),
nnet.Activation('relu'),
nnet.Linear(
n_out=50,
weight_scale=0.2,
weight_decay=0.004,
),
nnet.Activation('relu'),
nnet.Linear(
n_out=n_classes,
weight_scale=0.2,
weight_decay=0.004,
),
nnet.LogRegression(),
], )
# Train neural network
t0 = time.time()
nn.fit(X_train, y_train, learning_rate=0.1, max_iter=5, batch_size=64)
t1 = time.time()
print('Duration: %.1fs' % (t1 - t0))
# Evaluate on test data
error = nn.error(X_test, y_test)
print('Test error rate: %.4f' % error)
k_fold = KFold(n_splits=10)
result = []
for index, nf in enumerate(n_feats):
fold_result = []
print('*** Starting Test of feat [', n_feats[index], ']...')
# SETUP one-layer CONVnet
nn = nnet.NeuralNetwork(layers=[
nnet.Conv(
n_feats=nf,
filter_shape=(5, 5),
strides=(1, 1),
weight_scale=0.1,
),
nnet.Activation('relu'),
nnet.Flatten(),
nnet.Linear(
n_out=n_classes,
weight_scale=0.1,
),
nnet.LogRegression(),
], )
# TRAINING
for train_indices, valid_indices in k_fold.split(np.array(X_train)):
np.random.shuffle(train_indices)
#print(train_indices, valid_indices)
X_tr = X_train[train_indices, ...]
Y_tr = Y_train[train_indices, ...]
X_val = X_train[valid_indices, ...]
def optimize_filter(n_train_samples,
n_classes,
X_train,
Y_train,
split,
weight_decay=0.0):
train_idxs = np.random.randint(0, split - 1, n_train_samples)
n_feats = [2, 4, 6, 8, 12, 16] # for the second layer!
X_train = X_train[train_idxs, ...]
Y_train = Y_train[train_idxs, ...]
one_layer_result = []
for index, nf in enumerate(n_feats):
fold_result = []
print('*** Starting 1-layer test of feat [', nf, ']...')
# SETUP one-layer CONVnet
nn = nnet.NeuralNetwork(layers=[
nnet.Conv(
n_feats=nf,
filter_shape=(5, 5),
strides=(1, 1),
weight_scale=0.1,
weight_decay=weight_decay,
),
nnet.Activation('relu'),
nnet.Flatten(),
nnet.Linear(
n_out=n_classes,
weight_scale=0.1,
),
nnet.LogRegression(),
], )
# TRAINING
for train_indices, valid_indices in k_fold.split(np.array(X_train)):
np.random.shuffle(train_indices)
#print(train_indices, valid_indices)
X_tr = X_train[train_indices, ...]
Y_tr = Y_train[train_indices, ...]
X_val = X_train[valid_indices, ...]
Y_val = Y_train[valid_indices, ...]
# Train neural network
t0 = time.time()
# TODO: max_iter 50
nn.fit(X_tr, Y_tr, learning_rate=0.1, max_iter=50, batch_size=30)
t1 = time.time()
# Evaluate on test data
error = nn.error(X_val, Y_val)
fold_result.append(error)
print('Duration: %.1fs' % (t1 - t0))
print('Valid error rate: %.4f' % error)
# save the result for each n_feat
one_layer_result.append(np.mean(np.array(fold_result)))
best_one_layer = n_feats[np.argmin(one_layer_result)]
# Try two-layer CONVnet
two_layer_result = []
for index, nf in enumerate(n_feats):
fold_result = []
print('*** Starting 2-layers-test of feat [', nf, ']...')
# SETUP two-layers CONVnet
nn = nnet.NeuralNetwork(layers=[
nnet.Conv(
n_feats=best_one_layer,
filter_shape=(5, 5),
strides=(1, 1),
weight_scale=0.1,
weight_decay=weight_decay,
),
nnet.Activation('relu'),
nnet.Pool(
pool_shape=(2, 2),
strides=(2, 2),
mode='max',
),
nnet.Conv(
n_feats=nf,
filter_shape=(5, 5),
strides=(1, 1),
weight_scale=0.1,
weight_decay=weight_decay,
),
nnet.Activation('relu'),
nnet.Flatten(),
nnet.Linear(
n_out=n_classes,
weight_scale=0.1,
),
nnet.LogRegression(),
], )
# TRAINING
for train_indices, valid_indices in k_fold.split(np.array(X_train)):
np.random.shuffle(train_indices)
#print(train_indices, valid_indices)
X_tr = X_train[train_indices, ...]
Y_tr = Y_train[train_indices, ...]
X_val = X_train[valid_indices, ...]
Y_val = Y_train[valid_indices, ...]
# Train neural network
t0 = time.time()
# TODO: max_iter 50
nn.fit(X_tr, Y_tr, learning_rate=0.1, max_iter=50, batch_size=30)
t1 = time.time()
# Evaluate on test data
error = nn.error(X_val, Y_val)
fold_result.append(error)
print('Duration: %.1fs' % (t1 - t0))
print('Valid error rate: %.4f' % error)
# save the result for each n_feat
two_layer_result.append(np.mean(np.array(fold_result)))
best_two_layer = n_feats[np.argmin(two_layer_result)]
print('One-layer result :', one_layer_result)
print('Two-layer result :', two_layer_result)
print('Two-Layer Optimum N_feat Value :', best_one_layer, best_two_layer)
return best_one_layer, best_two_layer