def main(data_set="cifar10"):
if data_set == "cifar10":
# load cifar10 data set
train_images, train_labels, val_images, val_labels, test_images, test_labels = cfg.load_cifar10()
classes = cfg.CIFAR10_classes
elif data_set == "mnist":
mnist = input_data.read_data_sets(cfg.MNIST_PATH)
train_images = mnist.train.images
train_labels = mnist.train.labels
test_images = mnist.test.images
test_labels = mnist.test.labels
classes = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
np.random.seed(231)
small_data = {
'X_train': train_images[:1000],
'y_train': train_labels[:1000],
'X_val': val_images,
'y_val': val_labels,
}
data = {
'X_train': train_images,
'y_train': train_labels,
'X_val': val_images,
'y_val': val_labels,
}
dropout_cmp(small_data)
def main():
# load cifar10 data set
train_images, train_labels, val_images, val_labels, test_images, test_labels = cfg.load_cifar10(
)
classes = cfg.CIFAR10_classes
num_classes = len(classes)
# bias trick : append the bias dimension of ones
train_images = np.hstack(
[train_images, np.ones((train_images.shape[0], 1))])
val_images = np.hstack([val_images, np.ones((val_images.shape[0], 1))])
test_images = np.hstack([test_images, np.ones((test_images.shape[0], 1))])
# SVM Classifier
classifier = linear_svm()
# training
loss_hist = classifier.train(train_images, train_labels, num_classes)
# test
predict_labels = classifier.predict(test_images)
# accuracy
acc = np.mean(predict_labels == test_labels)
print("Accuracy :", acc)
# Visualization
plt.figure("training loss curve")
plt.plot(loss_hist)
plt.xlabel('Mini batch step (batch size :' + str(classifier.batch_size) +
')')
plt.ylabel('Loss value')
# Visualize the learned weights for each class.
w = classifier.W[:-1, :] # strip out the bias
w = w.reshape(32, 32, 3, 10)
w_min, w_max = np.min(w), np.max(w)
plt.figure("weight matrix visualization")
for i in range(10):
plt.subplot(2, 5, i + 1)
# Rescale the weights to be between 0 and 255
wimg = 255.0 * (w[:, :, :, i].squeeze() - w_min) / (w_max - w_min)
plt.imshow(wimg.astype('uint8'))
plt.axis('off')
plt.title(classes[i])
plt.show()
def main():
k_list = list(range(1, 10))
# load cifar10 data set
train_images, train_labels, val_images, val_labels, test_images, test_labels = cfg.load_cifar10(
)
classes = cfg.CIFAR10_classes
num_classes = len(classes)
# display some examples
#show_example(train_images, train_labels, classes, num_classes)
# Subsample the data for more efficient code execution in this exercise
num_train = 10000
mask = list(range(num_train))
train_images = train_images[mask]
train_labels = train_labels[mask]
num_test = 500
mask = list(range(num_test))
test_images = test_images[mask]
test_labels = test_labels[mask]
# Build k nearest neighbor classifier
classifier = knn()
# training
classifier.train(train_images, train_labels)
acc_list = []
for k in k_list:
# predict
predict_labels = classifier.predict(test_images, k=k)
# calc acc
acc = np.sum(
np.array(predict_labels == test_labels).astype(float)) / num_test
acc_list.append(acc)
print("k :", k, " Test accuracy :", acc)
plt.plot(k_list, acc_list, '-o')
plt.xlabel('k')
plt.ylabel('accuracy')
plt.show()
def main(data_set="cifar10"):
if data_set == "cifar10":
# load cifar10 data set
train_images, train_labels, val_images, val_labels, test_images, test_labels = cfg.load_cifar10(
)
classes = cfg.CIFAR10_classes
elif data_set == "mnist":
mnist = input_data.read_data_sets(cfg.MNIST_PATH)
train_images = mnist.train.images
train_labels = mnist.train.labels
test_images = mnist.test.images
test_labels = mnist.test.labels
classes = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
train_images = train_images.reshape(-1, 3, 32, 32)
val_images = val_images.reshape(-1, 3, 32, 32)
test_images = test_images.reshape(-1, 3, 32, 32)
data = {
'X_train': train_images,
'y_train': train_labels,
'X_val': val_images,
'y_val': val_labels,
}
np.random.seed(231)
test(data)
def main(data_set="cifar10"):
if data_set == "cifar10":
# load cifar10 data set
train_images, train_labels, val_images, val_labels, test_images, test_labels = cfg.load_cifar10(
)
classes = cfg.CIFAR10_classes
elif data_set == "mnist":
mnist = input_data.read_data_sets(cfg.MNIST_PATH)
train_images = mnist.train.images
train_labels = mnist.train.labels
test_images = mnist.test.images
test_labels = mnist.test.labels
classes = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
num_classes = len(classes)
#---------------two layer network---------------#
input_size = train_images.shape[1]
hidden_size = 50
np.random.seed(0)
net = two_layer_net(input_size, hidden_size, num_classes)
#net.gradiant_check(train_images, train_labels) # !!! will take too much time !!!
stats = net.train(train_images, train_labels, num_iters=20)
print('Final training loss: ', stats['loss_history'][-1])
print('Final training acc: ', stats['train_acc_history'][-1])
# plot the loss history
plt.subplot(2, 1, 1)
plt.plot(stats['loss_history'])
plt.title('Loss history')
plt.ylabel('Loss')
plt.subplot(2, 1, 2)
plt.plot(stats['train_acc_history'], label='train')
plt.title('Classification accuracy history')
plt.xlabel('Batch Epoch')
plt.ylabel('Clasification accuracy')
# visualize the weights of the network
show_net_weights(net, data_set)
plt.show()
def main(data_set="cifar10"):
if data_set == "cifar10":
# load cifar10 data set
train_images, train_labels, val_images, val_labels, test_images, test_labels = cfg.load_cifar10(
)
train_images = train_images.reshape(-1, 32, 32, 3)
val_images = val_images.reshape(-1, 32, 32, 3)
test_images = test_images.reshape(-1, 32, 32, 3)
elif data_set == "mnist":
mnist = input_data.read_data_sets(cfg.MNIST_PATH)
train_images = mnist.train.images
train_labels = mnist.train.labels
test_images = mnist.test.images
test_labels = mnist.test.labels
classes = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
# init network
net = model(input_dim=(32, 32, 3))
net.reg = 5e-5
# define optimizer
optimizer = tf.train.AdamOptimizer(5e-4).minimize(net.loss)
# run
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
save_name = 'tf.ckpt'
restore_name = save_name.split('.')[0] + '_final.ckpt'
print('Training')
run(sess, net, train_images, train_labels, batch_size=64, \
epochs=4, optimizer=optimizer, plot_losses=True, \
save_model=save_name#, load_model=restore_name,
)
print('Validation')
run(sess, net, val_images, val_labels, load_model=restore_name)
def main(data_set="cifar10"):
if data_set == "cifar10":
# load cifar10 data set
train_images, train_labels, val_images, val_labels, test_images, test_labels = cfg.load_cifar10(
)
classes = cfg.CIFAR10_classes
elif data_set == "mnist":
mnist = input_data.read_data_sets(cfg.MNIST_PATH)
train_images = mnist.train.images
train_labels = mnist.train.labels
test_images = mnist.test.images
test_labels = mnist.test.labels
classes = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
np.random.seed(231)
# Try training a very deep net with batchnorm
hidden_dims = [100, 100, 100, 100, 100]
num_train = 1000
small_data = {
'X_train': train_images[:num_train],
'y_train': train_labels[:num_train],
'X_val': val_images,
'y_val': val_labels,
}
bn_solvers = {}
solvers = {}
weight_scales = np.logspace(-4, 0, num=20)
for i, weight_scale in enumerate(weight_scales):
print('Running weight scale %d / %d' % (i + 1, len(weight_scales)))
bn_model = FullyConnectedNet(hidden_dims,
weight_scale=weight_scale,
use_batchnorm=True)
model = FullyConnectedNet(hidden_dims,
weight_scale=weight_scale,
use_batchnorm=False)
bn_solver = Solver(bn_model,
small_data,
num_epochs=10,
batch_size=50,
update_rule='adam',
optim_config={
'learning_rate': 1e-3,
},
verbose=False,
print_every=200)
bn_solver.train()
bn_solvers[weight_scale] = bn_solver
solver = Solver(model,
small_data,
num_epochs=10,
batch_size=50,
update_rule='adam',
optim_config={
'learning_rate': 1e-3,
},
verbose=False,
print_every=200)
solver.train()
solvers[weight_scale] = solver
# Plot results of weight scale experiment
best_train_accs, bn_best_train_accs = [], []
best_val_accs, bn_best_val_accs = [], []
final_train_loss, bn_final_train_loss = [], []
for ws in weight_scales:
best_train_accs.append(max(solvers[ws].train_acc_history))
bn_best_train_accs.append(max(bn_solvers[ws].train_acc_history))
best_val_accs.append(max(solvers[ws].val_acc_history))
bn_best_val_accs.append(max(bn_solvers[ws].val_acc_history))
final_train_loss.append(np.mean(solvers[ws].loss_history[-100:]))
bn_final_train_loss.append(np.mean(bn_solvers[ws].loss_history[-100:]))
plt.subplot(3, 1, 1)
plt.title('Best val accuracy vs weight initialization scale')
plt.xlabel('Weight initialization scale')
plt.ylabel('Best val accuracy')
plt.semilogx(weight_scales, best_val_accs, '-o', label='baseline')
plt.semilogx(weight_scales, bn_best_val_accs, '-o', label='batchnorm')
plt.legend(ncol=2, loc='lower right')
plt.subplot(3, 1, 2)
plt.title('Best train accuracy vs weight initialization scale')
plt.xlabel('Weight initialization scale')
plt.ylabel('Best training accuracy')
plt.semilogx(weight_scales, best_train_accs, '-o', label='baseline')
plt.semilogx(weight_scales, bn_best_train_accs, '-o', label='batchnorm')
plt.legend()
plt.subplot(3, 1, 3)
plt.title('Final training loss vs weight initialization scale')
plt.xlabel('Weight initialization scale')
plt.ylabel('Final training loss')
plt.semilogx(weight_scales, final_train_loss, '-o', label='baseline')
plt.semilogx(weight_scales, bn_final_train_loss, '-o', label='batchnorm')
plt.legend()
plt.gca().set_ylim(1.0, 3.5)
plt.gcf().set_size_inches(10, 15)
plt.show()