def visualizeBoundary(X, y, model):
#VISUALIZEBOUNDARY plots a non-linear decision boundary learned by the SVM
# VISUALIZEBOUNDARYLINEAR(X, y, model) plots a non-linear decision
# boundary learned by the SVM and overlays the data on it
# Plot the training data on top of the boundary
plotData(X, y)
# Make classification predictions over a grid of values
x1plot = np.linspace(min(X[:, 0]), max(X[:, 0]), 100)
x2plot = np.linspace(min(X[:, 1]), max(X[:, 1]), 100)
X1, X2 = np.meshgrid(x1plot, x2plot)
vals = np.zeros(X1.shape)
for i in range(X1.shape[1]):
this_X = np.stack((X1[:, i], X2[:, i]), axis=1)
vals[:, i] = svmPredict(model, this_X)
# Plot the SVM boundary
#hold on
plt.contour(X1, X2, vals, colors='y', linewidths=2)
plt.pcolormesh(X1,
X2,
vals,
cmap='YlGnBu',
alpha=0.25,
edgecolors='None',
lw=0)
plt.grid(False)
def visualizeBoundary(X, y, model):
"""
Plots a non-linear decision boundary learned by the SVM and overlays the data on it.
Parameters
----------
X : array_like
(m x 2) The training data with two features (to plot in a 2-D plane).
y : array_like
(m, ) The data labels.
model : dict
Dictionary of model variables learned by SVM.
"""
plotData(X, y)
# make classification predictions over a grid of values
x1plot = np.linspace(min(X[:, 0]), max(X[:, 0]), 100)
x2plot = np.linspace(min(X[:, 1]), max(X[:, 1]), 100)
X1, X2 = np.meshgrid(x1plot, x2plot)
vals = np.zeros(X1.shape)
for i in range(X1.shape[1]):
this_X = np.stack((X1[:, i], X2[:, i]), axis=1)
vals[:, i] = svmPredict(model, this_X)
plt.contour(X1, X2, vals, colors='y', linewidths=2)
plt.pcolormesh(X1,
X2,
vals,
cmap='YlGnBu',
alpha=0.25,
edgecolors='None',
lw=0)
plt.grid(False)
def visualizeBoundary(X, y, model):
# Plot the training data on top of the boundary
plotData(X,y.T[0])
# Make classification predictions over a grid of values
x1plot = np.linspace(np.min(X[:,0]), np.max(X[:,0]), 100).reshape(-1,1)
x2plot = np.linspace(np.min(X[:,1]), np.max(X[:,1]), 100).reshape(-1,1)
X1, X2 = np.meshgrid(x1plot, x2plot)
vals = np.zeros(X1.shape)
for i in range(X1.shape[1]):
this_X = np.vstack((X1[:,i], X2[:,i])).T
vals[:,i] = svmPredict(model, this_X).reshape(1,-1)
# Plot the SVM boundary
plt.contour(X1, X2, vals, colors='b', linewidths=1)
def dataset3Params(X, y, Xval, yval):
"""returns your choice of C and sigma. for Part 3 of the exercise
where you select the optimal (C, sigma) learning parameters to use for SVM
with RBF kernel
"""
# C, sigma = dataset3Params(X, y, Xval, yval) returns your choice of C and
# sigma. You should complete this function to return the optimal C and
# sigma based on a cross-validation set.
# You need to return the following variables correctly.
C = 1
sigma = 0.3
# ====================== YOUR CODE HERE ======================
# Instructions: Fill in this function to return the optimal C and sigma
# learning parameters found using the cross validation set.
# You can use svmPredict to predict the labels on the cross
# validation set. For example,
# predictions = svmPredict(model, Xval)
# will return the predictions on the cross validation set.
#
# Note: You can compute the prediction error using
# mean(double(predictions ~= yval))
#
values = np.array([0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30])
pred_error = np.zeros((len(values), len(values)))
for i, C_ in enumerate(values):
for j, sigma_ in enumerate(values):
gamma = 1 / (2 * sigma_**2)
model = SVC(
C=C_, kernel='rbf', gamma=gamma, max_iter=200).fit(X, y)
predictions = svmPredict(model, Xval)
pred_error[i, j] = np.mean(predictions != yval)
i, j = np.argwhere(pred_error == np.min(pred_error))[0]
C = values[i]
sigma = values[j]
# =========================================================================
return C, sigma
def visualizeBoundary(X, y, model):
#VISUALIZEBOUNDARY plots a non-linear decision boundary learned by the SVM
# VISUALIZEBOUNDARYLINEAR(X, y, model) plots a non-linear decision
# boundary learned by the SVM and overlays the data on it
# Plot the training data on top of the boundary
plotData(X, y)
# Make classification predictions over a grid of values
x1plot = linspace(min(X[:, 0]), max(X[:, 0]), 100)
x2plot = linspace(min(X[:, 1]), max(X[:, 1]), 100)
[X1, X2] = meshgrid(x1plot, x2plot)
vals = zeros(shape(X1))
for i in range(size(X1, 1)):
this_X = column_stack((X1[:, i], X2[:, i]))
vals[:, i] = svmPredict(model, this_X)
# Plot the SVM boundary
hold(True)
contour(X1, X2, vals, [0], color='b')
hold(False)
def visualizeBoundary(X, y, model):
#VISUALIZEBOUNDARY plots a non-linear decision boundary learned by the SVM
# VISUALIZEBOUNDARYLINEAR(X, y, model) plots a non-linear decision
# boundary learned by the SVM and overlays the data on it
# Plot the training data on top of the boundary
plotData(X, y)
# Make classification predictions over a grid of values
x1plot = linspace(min(X[:,0]), max(X[:,0]), 100)
x2plot = linspace(min(X[:,1]), max(X[:,1]), 100)
[X1, X2] = meshgrid(x1plot, x2plot)
vals = zeros(shape(X1))
for i in range(size(X1, 1)):
this_X = column_stack((X1[:, i], X2[:, i]))
vals[:, i] = svmPredict(model, this_X)
# Plot the SVM boundary
hold(True)
contour(X1, X2, vals, [0], color='b')
hold(False)
def visualizeBoundary(X, y, model):
"""plots a non-linear decision boundary learned by the
SVM and overlays the data on it"""
m = np.size(X, 0)
# Plot the training data on top of the boundary
plotData(X, y)
# Make classification predictions over a grid of values
x1plot = np.linspace(min(X[:, 0]), max(X[:, 0]), m)
x2plot = np.linspace(min(X[:, 1]), max(X[:, 1]), m)
X1, X2 = np.meshgrid(x1plot, x2plot)
vals = np.zeros((m, m))
for i in range(m):
this_X = np.c_[X1[:, i], X2[:, i]] # (863, 2)
vals[:, i] = svmPredict(model, this_X)
# Plot the SVM boundary
plt.contour(X1, X2, vals, colors='blue')
def dataset3Params(X, y, Xval, yval):
#EX6PARAMS returns your choice of C and sigma for Part 3 of the exercise
#where you select the optimal (C, sigma) learning parameters to use for SVM
#with RBF kernel
# [C, sigma] = EX6PARAMS(X, y, Xval, yval) returns your choice of C and
# sigma. You should complete this function to return the optimal C and
# sigma based on a cross-validation set.
#
# You need to return the following variables correctly.
#C = 1;
#sigma = 0.3;
# ====================== YOUR CODE HERE ======================
# Instructions: Fill in this function to return the optimal C and sigma
# learning parameters found using the cross validation set.
# You can use svmPredict to predict the labels on the cross
# validation set. For example,
# predictions = svmPredict(model, Xval);
# will return the predictions on the cross validation set.
#
# Note: You can compute the prediction error using
# mean(double(predictions ~= yval))
#
values = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30]
error = 10000
for i in range(len(values)):
C_val = values[i]
for j in range(len(values)):
sigma_val = values[j]
model = svmTrain(X, y, C_val, gaussianKernel, args=(sigma_val, ))
predictions = svmPredict(model, Xval)
error_val = np.mean(predictions != yval)
if error_val < error:
error, C, sigma = error_val, C_val, sigma_val\
# =========================================================================
return (C, sigma)
def dataset3Params(X, y, Xval, yval):
choice = np.array([0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30]).reshape(-1, 1)
minError = np.inf
curC = np.inf
cur_sigma = np.inf
for i in range(choice.shape[0]):
for j in range(choice.shape[0]):
func = lambda a, b: gaussianKernel(a, b, choice[j])
func.__name__ = 'gaussianKernel'
model = svmTrain(X, y, choice[i], func)
predictions = svmPredict(model, Xval)
error = np.mean(np.double(np.not_equal(predictions, yval)))
if error < minError:
minError = error
curC = choice[i]
cur_sigma = choice[j]
C = curC
sigma = cur_sigma
return C, sigma
def dataset3Params(X, y, Xval, yval):
choice = np.array([0.01,0.03,0.1,0.3,1,3,10,30]).reshape(-1,1)
minError = np.inf
curC = np.inf
cur_sigma = np.inf
for i in range(choice.shape[0]):
for j in range(choice.shape[0]):
func = lambda a, b: gaussianKernel(a, b, choice[j])
func.__name__ = 'gaussianKernel'
model = svmTrain(X, y, choice[i], func)
predictions = svmPredict(model, Xval)
error = np.mean(np.double(np.not_equal(predictions,yval)))
if error < minError:
minError = error
curC = choice[i]
cur_sigma = choice[j]
C = curC
sigma = cur_sigma
return C, sigma
# Load the Spam Email dataset
# You will have X, y in your environment
spamTrain = loadmat('spamTrain.mat')
# matrices are converted to float because matrix multiplication is
# MUCH slower for integer matrices in numpy
X = spamTrain['X'].astype(float32)
y = spamTrain['y'].ravel().astype(float32)
print '\nTraining Linear SVM (Spam Classification)'
print '(this may take 1 to 2 minutes) ...'
C = 0.1
model = svmTrain(X, y, C, linearKernel)
p = svmPredict(model, X)
print 'Training Accuracy: %f' % (mean(p == y) * 100)
## =================== Part 4: Test Spam Classification ================
# After training the classifier, we can evaluate it on a test set. We have
# included a test set in spamTest.mat
# Load the test dataset
# You will have Xtest, ytest in your environment
spamTest = loadmat('spamTest.mat')
Xtest = spamTest['Xtest'].astype(float32)
ytest = spamTest['ytest'].ravel().astype(float32)
print '\nEvaluating the trained Linear SVM on a test set ...'
# Load the Spam Email dataset
# You will have X, y in your environment
spamTrain = loadmat('spamTrain.mat')
# matrices are converted to float because matrix multiplication is
# MUCH slower for integer matrices in numpy
X = spamTrain['X'].astype(float32)
y = spamTrain['y'].ravel().astype(float32)
print '\nTraining Linear SVM (Spam Classification)'
print '(this may take 1 to 2 minutes) ...'
C = 0.1
model = svmTrain(X, y, C, linearKernel)
p = svmPredict(model, X)
print 'Training Accuracy: %f' % (mean(p == y) * 100)
## =================== Part 4: Test Spam Classification ================
# After training the classifier, we can evaluate it on a test set. We have
# included a test set in spamTest.mat
# Load the test dataset
# You will have Xtest, ytest in your environment
spamTest = loadmat('spamTest.mat')
Xtest = spamTest['Xtest'].astype(float32)
ytest = spamTest['ytest'].ravel().astype(float32)
print '\nEvaluating the trained Linear SVM on a test set ...'
def ex6_spam():
## Machine Learning Online Class
# Exercise 6 | Spam Classification with SVMs
#
# Instructions
# ------------
#
# This file contains code that helps you get started on the
# exercise. You will need to complete the following functions:
#
# gaussianKernel.m
# dataset3Params.m
# processEmail.m
# emailFeatures.m
#
# For this exercise, you will not need to change any code in this file,
# or any other files other than those mentioned above.
#
## Initialization
#clear ; close all; clc
## ==================== Part 1: Email Preprocessing ====================
# To use an SVM to classify emails into Spam v.s. Non-Spam, you first need
# to convert each email into a vector of features. In this part, you will
# implement the preprocessing steps for each email. You should
# complete the code in processEmail.m to produce a word indices vector
# for a given email.
print('\nPreprocessing sample email (emailSample1.txt)')
# Extract Features
file_contents = readFile('emailSample1.txt')
word_indices = processEmail(file_contents)
# Print Stats
print('Word Indices: ')
print(formatter(' %d', np.array(word_indices) + 1))
print('\n')
print('Program paused. Press enter to continue.')
#pause;
## ==================== Part 2: Feature Extraction ====================
# Now, you will convert each email into a vector of features in R^n.
# You should complete the code in emailFeatures.m to produce a feature
# vector for a given email.
print('\nExtracting features from sample email (emailSample1.txt)')
# Extract Features
file_contents = readFile('emailSample1.txt')
word_indices = processEmail(file_contents)
features = emailFeatures(word_indices)
# Print Stats
print('Length of feature vector: %d' % features.size)
print('Number of non-zero entries: %d' % np.sum(features > 0))
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 3: Train Linear SVM for Spam Classification ========
# In this section, you will train a linear classifier to determine if an
# email is Spam or Not-Spam.
# Load the Spam Email dataset
# You will have X, y in your environment
mat = scipy.io.loadmat('spamTrain.mat')
X = mat['X'].astype(float)
y = mat['y'][:, 0]
print('\nTraining Linear SVM (Spam Classification)\n')
print('(this may take 1 to 2 minutes) ...\n')
C = 0.1
model = svmTrain(X, y, C, linearKernel)
p = svmPredict(model, X)
print('Training Accuracy: %f' % (np.mean(p == y) * 100))
## =================== Part 4: Test Spam Classification ================
# After training the classifier, we can evaluate it on a test set. We have
# included a test set in spamTest.mat
# Load the test dataset
# You will have Xtest, ytest in your environment
mat = scipy.io.loadmat('spamTest.mat')
Xtest = mat['Xtest'].astype(float)
ytest = mat['ytest'][:, 0]
print('\nEvaluating the trained Linear SVM on a test set ...\n')
p = svmPredict(model, Xtest)
print('Test Accuracy: %f\n' % (np.mean(p == ytest) * 100))
#pause;
## ================= Part 5: Top Predictors of Spam ====================
# Since the model we are training is a linear SVM, we can inspect the
# weights learned by the model to understand better how it is determining
# whether an email is spam or not. The following code finds the words with
# the highest weights in the classifier. Informally, the classifier
# 'thinks' that these words are the most likely indicators of spam.
#
# Sort the weights and obtin the vocabulary list
idx = np.argsort(model['w'])
top_idx = idx[-15:][::-1]
vocabList = getVocabList()
print('\nTop predictors of spam: ')
for word, w in zip(np.array(vocabList)[top_idx], model['w'][top_idx]):
print(' %-15s (%f)' % (word, w))
#end
print('\n')
print('\nProgram paused. Press enter to continue.')
#pause;
## =================== Part 6: Try Your Own Emails =====================
# Now that you've trained the spam classifier, you can use it on your own
# emails! In the starter code, we have included spamSample1.txt,
# spamSample2.txt, emailSample1.txt and emailSample2.txt as examples.
# The following code reads in one of these emails and then uses your
# learned SVM classifier to determine whether the email is Spam or
# Not Spam
# Set the file to be read in (change this to spamSample2.txt,
# emailSample1.txt or emailSample2.txt to see different predictions on
# different emails types). Try your own emails as well!
filename = 'spamSample1.txt'
# Read and predict
file_contents = readFile(filename)
word_indices = processEmail(file_contents)
x = emailFeatures(word_indices)
p = svmPredict(model, x.ravel())
print('\nProcessed %s\n\nSpam Classification: %d' % (filename, p))
print('(1 indicates spam, 0 indicates not spam)\n')
