print "Calculating K"
sys.stdout.flush()
K = rbf_kernel(X, X, gamma=1 / (2 * sigma ** 2)).reshape(X.shape[0], X.shape[0])
scaler = preprocessing.StandardScaler().fit(K)
scaleK = scaler.transform(K)
KK = np.vstack([np.ones((scaleK.shape[0],)), scaleK]).T
Kval = rbf_kernel(Xval, X, gamma=1 / (2 * sigma ** 2)).reshape(Xval.shape[0], X.shape[0])
scaleKval = scaler.transform(Kval)
KKval = np.vstack([np.ones((scaleKval.shape[0],)), scaleKval.T]).T
print "Done!"
for C in Cvals:
print "sigma=", sigma, ", C=", C
sys.stdout.flush()
svm.theta = np.zeros((KK.shape[1],))
svm.train(KK, yy, learning_rate=lr, C=C, num_iters=it)
y_train_pred = svm.predict(KK)
acc = metrics.accuracy_score(yy, y_train_pred)
print "C=", C, "sigma=", sigma, "TrainAccuracy:", acc
y_pred = svm.predict(KKval)
acc = metrics.accuracy_score(yyval, y_pred)
print "C=", C, "sigma=", sigma, "ValAccuracy:", acc
if acc > best_acc:
best_acc = acc
best_C = C
best_sigma = sigma
print "Best C is ", best_C, ", best sigma is ", best_sigma
sys.stdout.flush()
# We train a linear SVM on the data set and the plot the learned #
# decision boundary #
############################################################################
############################################################################
# TODO #
# You will change this line to vary C. #
############################################################################
C = 1
############################################################################
svm = LinearSVM_twoclass()
svm.theta = np.zeros((XX.shape[1], ))
svm.train(XX, yy, learning_rate=1e-4, C=C, num_iters=50000, verbose=True)
# classify the training data
y_pred = svm.predict(XX)
print "Accuracy on training data = ", metrics.accuracy_score(yy, y_pred)
# visualize the decision boundary
utils.plot_decision_boundary(scaleX, y, svm, 'x1', 'x2', ['neg', 'pos'])
plt.savefig('fig2.pdf')
############################################################################
# Part 3: Training SVM with a kernel #
# We train an SVM with an RBF kernel on the data set and the plot the #
# We train a linear SVM on the data set and the plot the learned #
# decision boundary #
############################################################################
############################################################################
# TODO #
# You will change this line to vary C. #
############################################################################
C = 100.
############################################################################
svm = LinearSVM_twoclass()
svm.theta = np.zeros((XX.shape[1],))
svm.train(XX,yy,learning_rate=1e-4,C=C,num_iters=50000,verbose=True)
# classify the training data
y_pred = svm.predict(XX)
print "Accuracy on training data = ", metrics.accuracy_score(yy,y_pred)
# visualize the decision boundary
utils.plot_decision_boundary(scaleX,y,svm,'x1','x2',['neg','pos'])
plt.savefig('fig2.pdf')
############################################################################
# Part 3: Training SVM with a kernel #
# We train an SVM with an RBF kernel on the data set and the plot the #
scaleK = scaler.transform(K)
KK = np.hstack([np.ones((scaleK.shape[0], 1)), scaleK])
Kval = np.array([
utils.gaussian_kernel(x1, x2, sigma_val) for x1 in X_val
for x2 in X_train
]).reshape(X_val.shape[0], X_train.shape[0])
scaleKval = scaler.transform(Kval)
KKval = np.hstack([np.ones((scaleKval.shape[0], 1)), scaleKval])
for Cval in Cvals:
for learning_rate in learning_rates:
for iteration in iterations:
svm = LinearSVM_twoclass()
svm.theta = np.zeros((KK.shape[1], ))
svm.train(KK,
y_train,
learning_rate=learning_rate,
C=Cval,
num_iters=iteration,
verbose=True)
y_pred = svm.predict(KKval)
acc = np.mean(y_pred == y_val)
if acc > best_acc:
best_acc = acc
best_C = Cval
best_sigma = sigma_val
best_iteration = iteration
best_learning_rate = learning_rate
print "For gussian kernel: "
print "the best acc is ", best_acc
print "the best c is ", best_C
print "the best sigma is ", best_sigma
# svm.theta = np.zeros((XX.shape[1],)) # svm.train(XX, yy, learning_rate=lr, C=C, num_iters=it, verbose=False) # yp = svm.predict(XXval) # acc = metrics.accuracy_score(yyval, yp) # print acc # if acc > best_acc: # best_acc = acc # best_C = C # best_lr = lr # best_it = it # print 'best', best_C, best_acc best_C = 0.1 best_lr = 1e-1 best_it = 10000 svm.train(X, yy, learning_rate=1e-1, C=best_C, num_iters=best_it, verbose=True) ############################################################################# # end of your code # ############################################################################# ############################################################################# # what is the accuracy of the best model on the training data itself? # ############################################################################# # 2 lines of code expected y_pred = svm.predict(X) print "Accuracy of model on training data is: ", metrics.accuracy_score(yy, y_pred) #############################################################################
best_acc = 0
for sigma_val in sigma_vals:
K = np.array([utils.gaussian_kernel(x1,x2,sigma_val) for x1 in X_train for x2 in X_train]).reshape(X_train.shape[0],X_train.shape[0])
scaler = preprocessing.StandardScaler().fit(K)
scaleK = scaler.transform(K)
KK = np.hstack([np.ones((scaleK.shape[0],1)),scaleK])
Kval = np.array([utils.gaussian_kernel(x1,x2,sigma_val) for x1 in X_val for x2 in X_train]).reshape(X_val.shape[0],X_train.shape[0])
scaleKval = scaler.transform(Kval)
KKval = np.hstack([np.ones((scaleKval.shape[0],1)),scaleKval])
for Cval in Cvals:
for learning_rate in learning_rates:
for iteration in iterations:
svm = LinearSVM_twoclass()
svm.theta = np.zeros((KK.shape[1],))
svm.train(KK,y_train,learning_rate=learning_rate,C=Cval,num_iters=iteration,verbose=True)
y_pred = svm.predict(KKval)
acc = np.mean(y_pred == y_val)
if acc > best_acc:
best_acc = acc
best_C=Cval
best_sigma = sigma_val
best_iteration = iteration
best_learning_rate = learning_rate
print "For gussian kernel: "
print "the best acc is ", best_acc
print "the best c is ", best_C
print "the best sigma is ", best_sigma
print "the best learning rate is ", best_learning_rate
print "the best iteration is ", best_iteration
############################################################################# # your code for setting up the best SVM classifier for this dataset # # Design the training parameters for the SVM. # # What should the learning_rate be? What should C be? # # What should num_iters be? Should X be scaled? Should X be kernelized? # ############################################################################# # your experiments below svm = LinearSVM_twoclass() svm.theta = np.zeros((X.shape[1],)) C = 1 svm.train(X,yy,learning_rate=1e-1,C=C,num_iters=8000,verbose=True) ############################################################################# # end of your code # ############################################################################# ############################################################################# # what is the accuracy of the best model on the training data itself? # ############################################################################# # 2 lines of code expected y_pred = svm.predict(X) print "Accuracy of model on training data is: ", metrics.accuracy_score(yy,y_pred) #############################################################################
iters = list(np.array(range(1601)) * 5)
trace = {}
for sigma in [10]:
trace[sigma] = []
K = metrics.pairwise.rbf_kernel(X_train, X_train, 1 / sigma ** 2)
scaler = preprocessing.StandardScaler().fit(K)
scaleK = scaler.transform(K)
KK = np.vstack([np.ones((scaleK.shape[0],)), scaleK]).T
Kval = metrics.pairwise.rbf_kernel(X_val, X_train, 1 / sigma ** 2)
scaler_val = preprocessing.StandardScaler().fit(Kval)
scaleKval = scaler_val.transform(Kval)
KKval = np.vstack([np.ones((scaleKval.shape[0],)), scaleKval.T]).T
svm.theta = np.zeros((KK.shape[1],))
trace[sigma].append(metrics.accuracy_score(y_val, svm.predict(KKval)))
for i in range(0, 1600):
svm.train(KK, y_train, learning_rate=0.01, C=2, num_iters=5, verbose=True)
a = metrics.accuracy_score(y_train, svm.predict(KK))
b = metrics.accuracy_score(y_val, svm.predict(KKval))
trace[sigma].append(a)
print ("%.2f,%d,%f,%f\d" % (sigma, i, a, b))
plt.plot(iters, trace[sigma], label="sigma=%.2f" % sigma)
plt.xlabel("iterations")
plt.ylabel("Accurancy on validation set")
plt.legend(bbox_to_anchor=(0.9, 0.9))
plt.savefig("kernal_sigma.pdf")
# svm.train(X,yy,learning_rate=0.1,C=0.1,num_iters=2000,verbose=True)
#############################################################################
print "Calculating K"
sys.stdout.flush()
K = np.array([utils.gaussian_kernel(x1,x2,sigma) for x1 in X for x2 in X]).reshape(X.shape[0],X.shape[0])
scaler = preprocessing.StandardScaler().fit(K)
scaleK = scaler.transform(K)
KK = np.vstack([np.ones((scaleK.shape[0],)),scaleK]).T
Kval = np.array([utils.gaussian_kernel(x1,x2,sigma) for x1 in Xval for x2 in X]).reshape(Xval.shape[0], X.shape[0])
scaleKval = scaler.transform(Kval)
KKval = np.vstack([np.ones((scaleKval.shape[0],)),scaleKval.T]).T
print "Done!"
for C in Cvals:
print "sigma=", sigma, ", C=", C
sys.stdout.flush()
svm.theta = np.zeros((KK.shape[1],))
svm.train(KK,yy,learning_rate=1e-4,C=C,num_iters=2000)
y_pred = svm.predict(KKval)
acc = metrics.accuracy_score(yyval,y_pred)
if (acc > best_acc):
best_acc = acc
best_C = C
best_sigma = sigma
print "Best C is ", best_C, ", best sigma is ", best_sigma
sys.stdout.flush()
#############################################################################
# end of your code #
#############################################################################