# then compute the probabilities for each of the points and then visualize
# both the overall distribution and where each of the points falls in
# terms of that distribution.
#
print 'Visualizing Gaussian fit.\n'
# Estimate my and sigma2
mu, sigma2 = estimateGaussian(X)
# Returns the density of the multivariate normal at each data point (row)
# of X
p = multivariateGaussian(X, mu, sigma2)
# Visualize the fit
fig = figure()
visualizeFit(X, mu, sigma2)
xlabel('Latency (ms)')
ylabel('Throughput (mb/s)')
fig.show()
print 'Program paused. Press enter to continue.'
raw_input()
## ================== Part 3: Find Outliers ===================
# Now you will find a good epsilon threshold using a cross-validation set
# probabilities given the estimated Gaussian distribution
#
pval = multivariateGaussian(Xval, mu, sigma2)
epsilon, F1 = selectThreshold(yval, pval)
# We first estimate the parameters of our assumed Gaussian distribution,
# then compute the probabilities for each of the points and then visualize
# both the overall distribution and where each of the points falls in
# terms of that distribution.
#
print 'Visualizing Gaussian fit.'
# Estimate my and sigma2
mu, sigma2 = estimateGaussian(X)
# Returns the density of the multivariate normal at each data point (row)
# of X
p = multivariateGaussian(X, mu, sigma2)
# Visualize the fit
visualizeFit(X, mu, sigma2)
plt.xlabel('Latency (ms)')
plt.ylabel('Throughput (mb/s)')
show()
raw_input("Program paused. Press Enter to continue...")
## ================== Part 3: Find Outliers ===================
# Now you will find a good epsilon threshold using a cross-validation set
# probabilities given the estimated Gaussian distribution
#
pval = multivariateGaussian(Xval, mu, sigma2)
epsilon, F1 = selectThreshold(yval, pval)
print 'Best epsilon found using cross-validation: %e' % epsilon
def ex8():
## Machine Learning Online Class
# Exercise 8 | Anomaly Detection and Collaborative Filtering
#
# Instructions
# ------------
#
# This file contains code that helps you get started on the
# exercise. You will need to complete the following functions:
#
# estimateGaussian.m
# selectThreshold.m
# cofiCostFunc.m
#
# For this exercise, you will not need to change any code in this file,
# or any other files other than those mentioned above.
#
## Initialization
#clear ; close all; clc
## ================== Part 1: Load Example Dataset ===================
# We start this exercise by using a small dataset that is easy to
# visualize.
#
# Our example case consists of 2 network server statistics across
# several machines: the latency and throughput of each machine.
# This exercise will help us find possibly faulty (or very fast) machines.
#
print('Visualizing example dataset for outlier detection.\n')
# The following command loads the dataset. You should now have the
# variables X, Xval, yval in your environment
mat = scipy.io.loadmat('ex8data1.mat')
X = mat['X']
Xval = mat['Xval']
yval = mat['yval']
# Visualize the example dataset
plt.plot(X[:, 0], X[:, 1], 'bx')
plt.axis([0, 30, 0, 30])
plt.xlabel('Latency (ms)')
plt.ylabel('Throughput (mb/s)')
plt.savefig('figure1.png')
print('Program paused. Press enter to continue.')
#pause
## ================== Part 2: Estimate the dataset statistics ===================
# For this exercise, we assume a Gaussian distribution for the dataset.
#
# We first estimate the parameters of our assumed Gaussian distribution,
# then compute the probabilities for each of the points and then visualize
# both the overall distribution and where each of the points falls in
# terms of that distribution.
#
print('Visualizing Gaussian fit.\n')
# Estimate my and sigma2
mu, sigma2 = estimateGaussian(X)
# Returns the density of the multivariate normal at each data point (row)
# of X
p = multivariateGaussian(X, mu, sigma2)
# Visualize the fit
visualizeFit(X, mu, sigma2)
plt.xlabel('Latency (ms)')
plt.ylabel('Throughput (mb/s)')
plt.savefig('figure2.png')
print('Program paused. Press enter to continue.\n')
#pause
## ================== Part 3: Find Outliers ===================
# Now you will find a good epsilon threshold using a cross-validation set
# probabilities given the estimated Gaussian distribution
#
pval = multivariateGaussian(Xval, mu, sigma2)
epsilon, F1 = selectThreshold(yval, pval)
print('Best epsilon found using cross-validation: %e' % epsilon)
print('Best F1 on Cross Validation Set: %f' % F1)
print(' (you should see a value epsilon of about 8.99e-05)\n')
# Find the outliers in the training set and plot the
outliers = p < epsilon
# Draw a red circle around those outliers
#hold on
plt.plot(X[outliers, 0], X[outliers, 1], 'ro', lw=2, ms=10)
#hold off
plt.savefig('figure3.png')
print('Program paused. Press enter to continue.\n')
#pause
## ================== Part 4: Multidimensional Outliers ===================
# We will now use the code from the previous part and apply it to a
# harder problem in which more features describe each datapoint and only
# some features indicate whether a point is an outlier.
#
# Loads the second dataset. You should now have the
# variables X, Xval, yval in your environment
mat = scipy.io.loadmat('ex8data2.mat')
X = mat['X']
Xval = mat['Xval']
yval = mat['yval']
# Apply the same steps to the larger dataset
mu, sigma2 = estimateGaussian(X)
# Training set
p = multivariateGaussian(X, mu, sigma2)
# Cross-validation set
pval = multivariateGaussian(Xval, mu, sigma2)
# Find the best threshold
epsilon, F1 = selectThreshold(yval, pval)
print('Best epsilon found using cross-validation: %e' % epsilon)
print('Best F1 on Cross Validation Set: %f' % F1)
print('# Outliers found: %d ' % np.sum(p < epsilon))
print(' (you should see a value epsilon of about 1.38e-18)\n')