def dataset3Params(X, y, Xval, yval): C_values = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30] sigma_values = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30] scores = np.zeros(((len(C_values) * len(sigma_values)), 3)) counter = 0 for C in C_values: for sigma in sigma_values: gamma = 1 / (2 * np.square(sigma)) model = svmTrain(X, y, C, 'rbf', gamma) scores[counter, 0] = model.score(Xval, yval) scores[counter, 1] = C scores[counter, 2] = sigma counter = counter + 1 max_score = np.argmax(scores, axis=0) return scores[max_score[0], 1], scores[max_score[0], 2]
def dataset3Params(X, y, Xval, yval):
'''
C, sigma] = DATASET3PARAMS(X, y, Xval, yval) returns your choice of C and
sigma. You should complete this function to return the optimal C and
sigma based on a cross-validation set.
'''
from svmTrain import svmTrain
import numpy as np
import gaussianKernelGramMatrix as gkgm
# initialize prediction error which collect the errors for each of the
# 64 models together with its sigma and C hyper-parameters
predictionErrors = np.zeros((64, 3))
Counter = 0
for sigma in [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30]:
for C in [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30]:
# Train the model on training set
model = svmTrain(X, y, C=C, kernelFunction="gaussian", sigma=sigma)
# Evaluate model on the cross-validation set
Z = model.predict(gkgm.gaussianKernelGramMatrix(Xval, X))
# Compute the prediction errors
predictionErrors[Counter, 0] = np.mean(
(Z != yval.flatten()).astype(int))
# store corresponding sigma and C
predictionErrors[Counter, 1] = sigma
predictionErrors[Counter, 2] = C
# Move counter up
Counter += 1
# extract the row number with the lower error
# since its a tuple, we pick the first min found inside the tuple
minIndex = np.where(
predictionErrors[:, 0] == np.min(predictionErrors[:, 0]))[0][0]
sigma = predictionErrors[minIndex, 1]
C = predictionErrors[minIndex, 2]
return sigma, C
def dataset3Params(X, y, Xval, yval):
#EX6PARAMS returns your choice of C and sigma for Part 3 of the exercise
#where you select the optimal (C, sigma) learning parameters to use for SVM
#with RBF kernel
# [C, sigma] = EX6PARAMS(X, y, Xval, yval) returns your choice of C and
# sigma. You should complete this function to return the optimal C and
# sigma based on a cross-validation set.
#
# You need to return the following variables correctly.
#C = 1;
#sigma = 0.3;
# ====================== YOUR CODE HERE ======================
# Instructions: Fill in this function to return the optimal C and sigma
# learning parameters found using the cross validation set.
# You can use svmPredict to predict the labels on the cross
# validation set. For example,
# predictions = svmPredict(model, Xval);
# will return the predictions on the cross validation set.
#
# Note: You can compute the prediction error using
# mean(double(predictions ~= yval))
#
values = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30]
error = 10000
for i in range(len(values)):
C_val = values[i]
for j in range(len(values)):
sigma_val = values[j]
model = svmTrain(X, y, C_val, gaussianKernel, args=(sigma_val, ))
predictions = svmPredict(model, Xval)
error_val = np.mean(predictions != yval)
if error_val < error:
error, C, sigma = error_val, C_val, sigma_val\
# =========================================================================
return (C, sigma)
def dataset3Params(X, y, Xval, yval):
choice = np.array([0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30]).reshape(-1, 1)
minError = np.inf
curC = np.inf
cur_sigma = np.inf
for i in range(choice.shape[0]):
for j in range(choice.shape[0]):
func = lambda a, b: gaussianKernel(a, b, choice[j])
func.__name__ = 'gaussianKernel'
model = svmTrain(X, y, choice[i], func)
predictions = svmPredict(model, Xval)
error = np.mean(np.double(np.not_equal(predictions, yval)))
if error < minError:
minError = error
curC = choice[i]
cur_sigma = choice[j]
C = curC
sigma = cur_sigma
return C, sigma
def dataset3Params(X, y, Xval, yval):
choice = np.array([0.01,0.03,0.1,0.3,1,3,10,30]).reshape(-1,1)
minError = np.inf
curC = np.inf
cur_sigma = np.inf
for i in range(choice.shape[0]):
for j in range(choice.shape[0]):
func = lambda a, b: gaussianKernel(a, b, choice[j])
func.__name__ = 'gaussianKernel'
model = svmTrain(X, y, choice[i], func)
predictions = svmPredict(model, Xval)
error = np.mean(np.double(np.not_equal(predictions,yval)))
if error < minError:
minError = error
curC = choice[i]
cur_sigma = choice[j]
C = curC
sigma = cur_sigma
return C, sigma
## =========== Part 3: Train Linear SVM for Spam Classification ========
# In this section, you will train a linear classifier to determine if an
# email is Spam or Not-Spam.
# Load the Spam Email dataset
# You will have X, y in your environment
mat = scipy.io.loadmat('spamTrain.mat')
X = mat["X"]
y = mat["y"]
print('Training Linear SVM (Spam Classification)')
print('(this may take 1 to 2 minutes) ...')
C = 0.1
model = svmt.svmTrain(X, y, C, "linear")
p = model.predict(X)
raw_input('Training Accuracy: {:f}'.format( np.mean((p == y).astype(int)) * 100 ))
## =================== Part 4: Test Spam Classification ================
# After training the classifier, we can evaluate it on a test set. We have
# included a test set in spamTest.mat
# Load the test dataset
# You will have Xtest, ytest in your environment
mat = scipy.io.loadmat('spamTest.mat')
Xtest = mat["Xtest"]
ytest = mat["ytest"]
# The following code will train a linear SVM on the dataset and plot the
# decision boundary learned.
#
# Load from ex6data1:
# You will have X, y in your environment
data = sio.loadmat('ex6data1.mat')
X = data['X']; y = data['y']
y = y.astype(int)
print('\nTraining Linear SVM ...\n')
# You should try to change the C value below and see how the decision
# boundary varies (e.g., try C = 1000)
C = 1
model = svmTrain(X, y, C, linearKernel, 1e-3, 20)
print(model)
visualizeBoundaryLinear(X, y, model)
input('Program paused. Press enter to continue.\n')
## =============== Part 3: Implementing Gaussian Kernel ===============
# You will now implement the Gaussian kernel to use
# with the SVM. You should complete the code in gaussianKernel.m
#
print('\nEvaluating the Gaussian Kernel ...\n')
x1 = np.array([1,2,1]); x2 = np.array([0,4,-1]); sigma = 2
sim = gaussianKernel(x1, x2, sigma)
print('Gaussian Kernel between x1 = [1; 2; 1], x2 = [0; 4; -1], sigma = 0.5 :' \
# In this section, you will train a linear classifier to determine if an
# email is Spam or Not-Spam.
# Load the Spam Email dataset
# You will have X, y in your environment
spamTrain = loadmat('spamTrain.mat')
# matrices are converted to float because matrix multiplication is
# MUCH slower for integer matrices in numpy
X = spamTrain['X'].astype(float32)
y = spamTrain['y'].ravel().astype(float32)
print '\nTraining Linear SVM (Spam Classification)'
print '(this may take 1 to 2 minutes) ...'
C = 0.1
model = svmTrain(X, y, C, linearKernel)
p = svmPredict(model, X)
print 'Training Accuracy: %f' % (mean(p == y) * 100)
## =================== Part 4: Test Spam Classification ================
# After training the classifier, we can evaluate it on a test set. We have
# included a test set in spamTest.mat
# Load the test dataset
# You will have Xtest, ytest in your environment
spamTest = loadmat('spamTest.mat')
Xtest = spamTest['Xtest'].astype(float32)
ytest = spamTest['ytest'].ravel().astype(float32)
# In this section, you will train a linear classifier to determine if an
# email is Spam or Not-Spam.
# Load the Spam Email dataset
# You will have X, y in your environment
mat = scipy.io.loadmat('spamTrain.mat')
X = mat["X"]
y = mat["y"]
y = y.flatten()
print('Training Linear SVM (Spam Classification)')
print('(this may take 1 to 2 minutes) ...')
C = 0.1
model = svmt.svmTrain(X, y, C, "linear")
p = model.predict(X)
raw_input('Training Accuracy: {:f}'.format( np.mean((p == y).astype(int)) * 100 ))
## =================== Part 4: Test Spam Classification ================
# After training the classifier, we can evaluate it on a test set. We have
# included a test set in spamTest.mat
# Load the test dataset
# You will have Xtest, ytest in your environment
mat = scipy.io.loadmat('spamTest.mat')
Xtest = mat["Xtest"]
ytest = mat["ytest"]
# The following code will train a linear SVM on the dataset and plot the
# decision boundary learned.
#
# Load from ex6data1:
# You will have X, y in your environment
ex6data1 = loadmat('ex6data1.mat')
X = ex6data1['X']
y = ex6data1['y'].ravel().astype(int8)
print '\nTraining Linear SVM ...'
# You should try to change the C value below and see how the decision
# boundary varies (e.g., try C = 1000)
C = 1.0
model = svmTrain(X, y, C, linearKernel, 1e-3, 20)
fig = figure()
visualizeBoundaryLinear(X, y, model)
fig.show()
print 'Program paused. Press enter to continue.'
raw_input()
## =============== Part 3: Implementing Gaussian Kernel ===============
# You will now implement the Gaussian kernel to use
# with the SVM. You should complete the code in gaussianKernel.m
#
print '\nEvaluating the Gaussian Kernel ...'
x1 = array([1, 2, 1])
x2 = array([0, 4, -1])
def ex6_spam():
## Machine Learning Online Class
# Exercise 6 | Spam Classification with SVMs
#
# Instructions
# ------------
#
# This file contains code that helps you get started on the
# exercise. You will need to complete the following functions:
#
# gaussianKernel.m
# dataset3Params.m
# processEmail.m
# emailFeatures.m
#
# For this exercise, you will not need to change any code in this file,
# or any other files other than those mentioned above.
#
## Initialization
#clear ; close all; clc
## ==================== Part 1: Email Preprocessing ====================
# To use an SVM to classify emails into Spam v.s. Non-Spam, you first need
# to convert each email into a vector of features. In this part, you will
# implement the preprocessing steps for each email. You should
# complete the code in processEmail.m to produce a word indices vector
# for a given email.
print('\nPreprocessing sample email (emailSample1.txt)')
# Extract Features
file_contents = readFile('emailSample1.txt')
word_indices = processEmail(file_contents)
# Print Stats
print('Word Indices: ')
print(formatter(' %d', np.array(word_indices) + 1))
print('\n')
print('Program paused. Press enter to continue.')
#pause;
## ==================== Part 2: Feature Extraction ====================
# Now, you will convert each email into a vector of features in R^n.
# You should complete the code in emailFeatures.m to produce a feature
# vector for a given email.
print('\nExtracting features from sample email (emailSample1.txt)')
# Extract Features
file_contents = readFile('emailSample1.txt')
word_indices = processEmail(file_contents)
features = emailFeatures(word_indices)
# Print Stats
print('Length of feature vector: %d' % features.size)
print('Number of non-zero entries: %d' % np.sum(features > 0))
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 3: Train Linear SVM for Spam Classification ========
# In this section, you will train a linear classifier to determine if an
# email is Spam or Not-Spam.
# Load the Spam Email dataset
# You will have X, y in your environment
mat = scipy.io.loadmat('spamTrain.mat')
X = mat['X'].astype(float)
y = mat['y'][:, 0]
print('\nTraining Linear SVM (Spam Classification)\n')
print('(this may take 1 to 2 minutes) ...\n')
C = 0.1
model = svmTrain(X, y, C, linearKernel)
p = svmPredict(model, X)
print('Training Accuracy: %f' % (np.mean(p == y) * 100))
## =================== Part 4: Test Spam Classification ================
# After training the classifier, we can evaluate it on a test set. We have
# included a test set in spamTest.mat
# Load the test dataset
# You will have Xtest, ytest in your environment
mat = scipy.io.loadmat('spamTest.mat')
Xtest = mat['Xtest'].astype(float)
ytest = mat['ytest'][:, 0]
print('\nEvaluating the trained Linear SVM on a test set ...\n')
p = svmPredict(model, Xtest)
print('Test Accuracy: %f\n' % (np.mean(p == ytest) * 100))
#pause;
## ================= Part 5: Top Predictors of Spam ====================
# Since the model we are training is a linear SVM, we can inspect the
# weights learned by the model to understand better how it is determining
# whether an email is spam or not. The following code finds the words with
# the highest weights in the classifier. Informally, the classifier
# 'thinks' that these words are the most likely indicators of spam.
#
# Sort the weights and obtin the vocabulary list
idx = np.argsort(model['w'])
top_idx = idx[-15:][::-1]
vocabList = getVocabList()
print('\nTop predictors of spam: ')
for word, w in zip(np.array(vocabList)[top_idx], model['w'][top_idx]):
print(' %-15s (%f)' % (word, w))
#end
print('\n')
print('\nProgram paused. Press enter to continue.')
#pause;
## =================== Part 6: Try Your Own Emails =====================
# Now that you've trained the spam classifier, you can use it on your own
# emails! In the starter code, we have included spamSample1.txt,
# spamSample2.txt, emailSample1.txt and emailSample2.txt as examples.
# The following code reads in one of these emails and then uses your
# learned SVM classifier to determine whether the email is Spam or
# Not Spam
# Set the file to be read in (change this to spamSample2.txt,
# emailSample1.txt or emailSample2.txt to see different predictions on
# different emails types). Try your own emails as well!
filename = 'spamSample1.txt'
# Read and predict
file_contents = readFile(filename)
word_indices = processEmail(file_contents)
x = emailFeatures(word_indices)
p = svmPredict(model, x.ravel())
print('\nProcessed %s\n\nSpam Classification: %d' % (filename, p))
print('(1 indicates spam, 0 indicates not spam)\n')
file.close()
#print(file_contents)
word_indices = processEmail(file_contents)
#print(word_indices)
features = emailFeatures(word_indices)
print('Length of feature vector: ', len(features))
print('Number of non-zero entries: ', np.sum(features > 0))
# =========== Part 3: Train Linear SVM for Spam Classification ========
data = loadmat("data/spamTrain.mat")
X = data['X']
y = data['y']
C = 0.1
model = svmTrain(X, y, C, 'linear')
p = model.predict(X)
print('Training Accuracy: ', np.mean(np.double(p == y.ravel())) * 100)
# =================== Part 4: Test Spam Classification ================
data = loadmat("data/spamTest.mat")
Xtest = data['Xtest']
ytest = data['ytest']
p = model.predict(Xtest)
print('Test Accuracy: ', np.mean(np.double(p == ytest.ravel())) * 100)
# ================= Part 5: Top Predictors of Spam ====================
w = model.coef_[0]
idx = np.argsort(w)[::-1][:15]
def dataset3Params(X, y, Xval, yval):
#EX6PARAMS returns your choice of C and sigma for Part 3 of the exercise
#where you select the optimal (C, sigma) learning parameters to use for SVM
#with RBF kernel
# [C, sigma] = EX6PARAMS(X, y, Xval, yval) returns your choice of C and
# sigma. You should complete this function to return the optimal C and
# sigma based on a cross-validation set.
#
# You need to return the following variables correctly.
sigma = 0.3
C = 1
# ====================== YOUR CODE HERE ======================
# Instructions: Fill in this function to return the optimal C and sigma
# learning parameters found using the cross validation set.
# You can use svmPredict to predict the labels on the cross
# validation set. For example,
# predictions = svmPredict(model, Xval)
# will return the predictions on the cross validation set.
#
# Note: You can compute the prediction error using
# mean(double(predictions ~= yval))
#
### determining best C and sigma
# need x1 and x2, copied from ex6.py
x1 = [1, 2, 1]
x2 = [0, 4, -1]
# only uncomment if similar lines are uncommented on svmTrain.py
# yval = yval.astype("int32")
# yval[yval==0] = -1
# vector with all predictions from SVM
predictionErrors = np.zeros((64, 3))
predictionsCounter = 0
# iterate over values of sigma and C
for sigma in [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30]:
for C in [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30]:
print "trying sigma={:.2f}, C={:.2f}".format(sigma, C)
# train model on training corpus with current sigma and C
model = svmt.svmTrain(X, y, C, "gaussian", sigma=sigma)
# compute predictions on cross-validation set
predictions = model.predict(gkgm.gaussianKernelGramMatrix(Xval, X))
# compute prediction errors on cross-validation set
predictionErrors[predictionsCounter, 0] = np.mean(
(predictions != yval).astype(int))
# store corresponding C and sigma
predictionErrors[predictionsCounter, 1] = sigma
predictionErrors[predictionsCounter, 2] = C
# move counter up by one
predictionsCounter = predictionsCounter + 1
print(predictionErrors)
# calculate mins of columns with their indexes
row = predictionErrors.argmin(axis=0)
m = np.zeros(row.shape)
for i in xrange(len(m)):
m[i] = predictionErrors[row[i]][i]
# note that row[0] is the index of the min of the first column
# and that the first column corresponds to the error,
# so the row at predictionErrors(row(1),:) has best C and sigma
print(predictionErrors[row[0], 1])
print(predictionErrors[row[0], 2])
# get C and sigma form such row
sigma = predictionErrors[row[0], 1]
C = predictionErrors[row[0], 2]
return C, sigma
X, y = data['X'], data['y'][:, 0]
plotData(X, y)
plt.show(block=False)
input('Program paused. Press enter to continue.')
# ==================== Part 2: Training Linear SVM ====================
# The following code will train a linear SVM on the dataset and plot the
# decision boundary learned.
print('Training Linear SVM ...\n')
# You should try to change the C value below and see how the decision
# boundary varies (e.g., try C = 1000)
C = 1
#clf = svm.SVC(C=C, kernel='linear', tol=1e-3, max_iter=20)
#model = clf.fit(X, y.ravel())
model = svmTrain(X, y, C, linearKernel, 1e-3, 20)
visualizeBoundaryLinear(X, y, model)
plt.show(block=False)
input('Program paused. Press enter to continue.')
# =============== Part 3: Implementing Gaussian Kernel ===============
# You will now implement the Gaussian kernel to use
# with the SVM. You should complete the code in gaussianKernel.py
print('Evaluating the Gaussian Kernel ...')
x1 = np.array([1, 2, 1])
x2 = np.array([0, 4, -1])
sigma = 2
sim = gaussianKernel(x1, x2, sigma)
print('Gaussian Kernel between x1 = [1; 2; 1], x2 = [0; 4; -1],',
'sigma = 0.5 : %f' % sim)
print('(this value should be about 0.324652)\n')
# decision boundary learned.
#
# Load from ex6data1:
# You will have X, y in your environment
data = sio.loadmat('ex6data1.mat')
X = data['X']
y = data['y']
y = y.astype(int)
print('\nTraining Linear SVM ...\n')
# You should try to change the C value below and see how the decision
# boundary varies (e.g., try C = 1000)
C = 1
model = svmTrain(X, y, C, linearKernel, 1e-3, 20)
print(model)
visualizeBoundaryLinear(X, y, model)
input('Program paused. Press enter to continue.\n')
## =============== Part 3: Implementing Gaussian Kernel ===============
# You will now implement the Gaussian kernel to use
# with the SVM. You should complete the code in gaussianKernel.m
#
print('\nEvaluating the Gaussian Kernel ...\n')
x1 = np.array([1, 2, 1])
x2 = np.array([0, 4, -1])
sigma = 2
sim = gaussianKernel(x1, x2, sigma)
def ex6():
# Machine Learning Online Class
# Exercise 6 | Support Vector Machines
#
# Instructions
# ------------
#
# This file contains code that helps you get started on the
# exercise. You will need to complete the following functions:
#
# gaussianKernel.m
# dataset3Params.m
# processEmail.m
# emailFeatures.m
#
# For this exercise, you will not need to change any code in this file,
# or any other files other than those mentioned above.
#
# Initialization
#clear ; close all; clc
# =============== Part 1: Loading and Visualizing Data ================
# We start the exercise by first loading and visualizing the dataset.
# The following code will load the dataset into your environment and plot
# the data.
#
print('Loading and Visualizing Data ...')
# Load from ex6data1:
# You will have X, y in your environment
mat = scipy.io.loadmat('ex6data1.mat')
X = mat['X'].astype(float)
y = mat['y'][:, 0]
# Plot training data
plotData(X, y)
plt.savefig('figure1.png')
print('Program paused. Press enter to continue.')
#pause;
# ==================== Part 2: Training Linear SVM ====================
# The following code will train a linear SVM on the dataset and plot the
# decision boundary learned.
#
# Load from ex6data1:
# You will have X, y in your environment
mat = scipy.io.loadmat('ex6data1.mat')
X = mat['X'].astype(float)
y = mat['y'][:, 0]
print('\nTraining Linear SVM ...')
# You should try to change the C value below and see how the decision
# boundary varies (e.g., try C = 1000)
C = 1
model = svmTrain(X, y, C, linearKernel, 1e-3, 20)
visualizeBoundaryLinear(X, y, model)
plt.savefig('figure2.png')
print('Program paused. Press enter to continue.')
#pause;
# =============== Part 3: Implementing Gaussian Kernel ===============
# You will now implement the Gaussian kernel to use
# with the SVM. You should complete the code in gaussianKernel.m
#
print('\nEvaluating the Gaussian Kernel ...')
x1 = np.array([
1,
2,
1,
])
x2 = np.array([0, 4, -1])
sigma = 2
sim = gaussianKernel(x1, x2, sigma)
print(
'Gaussian Kernel between x1 = [1; 2; 1], x2 = [0; 4; -1], sigma = 0.5 :\n\t%f\n(this value should be about 0.324652)'
% sim)
print('Program paused. Press enter to continue.')
#pause;
# =============== Part 4: Visualizing Dataset 2 ================
# The following code will load the next dataset into your environment and
# plot the data.
#
fig = plt.figure()
print('Loading and Visualizing Data ...')
# Load from ex6data2:
# You will have X, y in your environment
mat = scipy.io.loadmat('ex6data2.mat')
X = mat['X'].astype(float)
y = mat['y'][:, 0]
# Plot training data
plotData(X, y)
plt.savefig('figure3.png')
print('Program paused. Press enter to continue.')
#pause;
# ========== Part 5: Training SVM with RBF Kernel (Dataset 2) ==========
# After you have implemented the kernel, we can now use it to train the
# SVM classifier.
#
print('\nTraining SVM with RBF Kernel (this may take 1 to 2 minutes) ...')
# Load from ex6data2:
# You will have X, y in your environment
mat = scipy.io.loadmat('ex6data2.mat')
X = mat['X'].astype(float)
y = mat['y'][:, 0]
# SVM Parameters
C = 1
sigma = 0.1
# We set the tolerance and max_passes lower here so that the code will run
# faster. However, in practice, you will want to run the training to
# convergence.
model = svmTrain(X, y, C, gaussianKernel, args=(sigma, ))
visualizeBoundary(X, y, model)
plt.savefig('figure4.png')
print('Program paused. Press enter to continue.')
#pause;
# =============== Part 6: Visualizing Dataset 3 ================
# The following code will load the next dataset into your environment and
# plot the data.
#
fig = plt.figure()
print('Loading and Visualizing Data ...')
# Load from ex6data3:
# You will have X, y in your environment
mat = scipy.io.loadmat('ex6data3.mat')
X = mat['X'].astype(float)
y = mat['y'][:, 0]
# Plot training data
plotData(X, y)
plt.savefig('figure5.png')
print('Program paused. Press enter to continue.')
#pause;
# ========== Part 7: Training SVM with RBF Kernel (Dataset 3) ==========
# This is a different dataset that you can use to experiment with. Try
# different values of C and sigma here.
#
# Load from ex6data3:
# You will have X, y in your environment
mat = scipy.io.loadmat('ex6data3.mat')
X = mat['X'].astype(float)
y = mat['y'][:, 0]
Xval = mat['Xval'].astype(float)
yval = mat['yval'][:, 0]
# Try different SVM Parameters here
C, sigma = dataset3Params(X, y, Xval, yval)
# Train the SVM
model = svmTrain(X, y, C, gaussianKernel, args=(sigma, ))
visualizeBoundary(X, y, model)
plt.savefig('figure6.png')
print('Program paused. Press enter to continue.')
def dataset3Params(X, y, Xval, yval):
#EX6PARAMS returns your choice of C and sigma for Part 3 of the exercise
#where you select the optimal (C, sigma) learning parameters to use for SVM
#with RBF kernel
# [C, sigma] = EX6PARAMS(X, y, Xval, yval) returns your choice of C and
# sigma. You should complete this function to return the optimal C and
# sigma based on a cross-validation set.
#
# You need to return the following variables correctly.
sigma = 0.3
C = 1
# ====================== YOUR CODE HERE ======================
# Instructions: Fill in this function to return the optimal C and sigma
# learning parameters found using the cross validation set.
# You can use svmPredict to predict the labels on the cross
# validation set. For example,
# predictions = svmPredict(model, Xval)
# will return the predictions on the cross validation set.
#
# Note: You can compute the prediction error using
# mean(double(predictions ~= yval))
#
### determining best C and sigma
# need x1 and x2, copied from ex6.py
x1 = [1, 2, 1]
x2 = [0, 4, -1]
# only uncomment if similar lines are uncommented on svmTrain.py
# yval = yval.astype("int32")
# yval[yval==0] = -1
# vector with all predictions from SVM
predictionErrors = np.zeros((64,3))
predictionsCounter = 0
# iterate over values of sigma and C
for sigma in [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30]:
for C in [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30]:
print "trying sigma={:.2f}, C={:.2f}".format(sigma, C)
# train model on training corpus with current sigma and C
model = svmt.svmTrain(X, y, C, "gaussian", sigma=sigma)
# compute predictions on cross-validation set
predictions = model.predict(gkgm.gaussianKernelGramMatrix(Xval, X))
# compute prediction errors on cross-validation set
predictionErrors[predictionsCounter,0] = np.mean((predictions != yval).astype(int))
# store corresponding C and sigma
predictionErrors[predictionsCounter,1] = sigma
predictionErrors[predictionsCounter,2] = C
# move counter up by one
predictionsCounter = predictionsCounter + 1
print(predictionErrors)
# calculate mins of columns with their indexes
row = predictionErrors.argmin(axis=0)
m = np.zeros(row.shape)
for i in xrange(len(m)):
m[i] = predictionErrors[row[i]][i]
# note that row[0] is the index of the min of the first column
# and that the first column corresponds to the error,
# so the row at predictionErrors(row(1),:) has best C and sigma
print(predictionErrors[row[0],1])
print(predictionErrors[row[0],2])
# get C and sigma form such row
sigma = predictionErrors[row[0],1]
C = predictionErrors[row[0],2]
return C, sigma
# =============== Part 4: Visualizing Dataset 2 ================
data2 = loadmat("data/ex6data2.mat")
X = data2['X']
y = data2['y']
#plt = plotData(X, y)
#plt.show()
C = 1
sigma = 0.1
# in sklearn SVC, gamma = 1/2*sigma^2
model = svmTrain(X, y, C, 'rbf', 1 / (2 * np.square(sigma)))
visualizeBoundary(X, y, model)
"""
# =============== Part 6: Visualizing Dataset 3 ================
data3 = loadmat("data/ex6data3.mat")
X = data3['X']
y = data3['y']
Xval = data3['Xval']
yval = data3['yval']
#plt = plotData(X, y)
#plt.show()
C, sigma = dataset3Params(X, y, Xval, yval)
#print('C=', C, 'Sigma=', sigma)
model = svmTrain(X, y, C, 'rbf', 1 / (2 * np.square(sigma)))
visualizeBoundary(X, y, model)
# The following code will train a linear SVM on the dataset and plot the
# decision boundary learned.
#
# Load from ex6data1:
# You will have X, y in your environment
ex6data1 = loadmat('ex6data1.mat')
X = ex6data1['X']
y = ex6data1['y'].ravel().astype(int8)
print '\nTraining Linear SVM ...'
# You should try to change the C value below and see how the decision
# boundary varies (e.g., try C = 1000)
C = 1.0
model = svmTrain(X, y, C, linearKernel, 1e-3, 20)
fig = figure()
visualizeBoundaryLinear(X, y, model)
fig.show()
print 'Program paused. Press enter to continue.'
raw_input()
## =============== Part 3: Implementing Gaussian Kernel ===============
# You will now implement the Gaussian kernel to use
# with the SVM. You should complete the code in gaussianKernel.m
#
print '\nEvaluating the Gaussian Kernel ...'
x1 = array([1, 2, 1])
x2 = array([0, 4, -1])
# In this section, you will train a linear classifier to determine if an
# email is Spam or Not-Spam.
# Load the Spam Email dataset
# You will have X, y in your environment
spamTrain = loadmat('spamTrain.mat')
# matrices are converted to float because matrix multiplication is
# MUCH slower for integer matrices in numpy
X = spamTrain['X'].astype(float32)
y = spamTrain['y'].ravel().astype(float32)
print '\nTraining Linear SVM (Spam Classification)'
print '(this may take 1 to 2 minutes) ...'
C = 0.1
model = svmTrain(X, y, C, linearKernel)
p = svmPredict(model, X)
print 'Training Accuracy: %f' % (mean(p == y) * 100)
## =================== Part 4: Test Spam Classification ================
# After training the classifier, we can evaluate it on a test set. We have
# included a test set in spamTest.mat
# Load the test dataset
# You will have Xtest, ytest in your environment
spamTest = loadmat('spamTest.mat')
Xtest = spamTest['Xtest'].astype(float32)
ytest = spamTest['ytest'].ravel().astype(float32)
# The following code will train a linear SVM on the dataset and plot the
# decision boundary learned.
#
# Load from ex6data1:
# You will have X, y in your environment
mat = scipy.io.loadmat('ex6data1.mat')
X = mat["X"]
y = mat["y"]
print('Training Linear SVM ...')
# You should try to change the C value below and see how the decision
# boundary varies (e.g., try C = 1000)
C = 1
model = svmt.svmTrain(X, y, C, "linear", 1e-3, 20)
plt.close()
vbl.visualizeBoundaryLinear(X, y, model)
raw_input('Program paused. Press enter to continue.')
## =============== Part 3: Implementing Gaussian Kernel ===============
# You will now implement the Gaussian kernel to use
# with the SVM. You should complete the code in gaussianKernel.m
#
print('Evaluating the Gaussian Kernel ...')
x1 = np.array([1, 2, 1])
x2 = np.array([0, 4, -1])
sigma = 2
sim = gk.gaussianKernel(x1, x2, sigma)
