def trainKMeans(X, threshold=100, max_clusters=10):
final_cost = -1
update = True
final_clusters: int
final_centroids_history: list
final_idx: np.ndarray
costs = []
for CLUSTERS in range(1, max_clusters + 1):
# Centroids List for plotting history
centroids_history = []
# Initializing Centroids
centroids = initializeCentroids(X, CLUSTERS)
centroids_history.append(centroids)
prev_cost = -1
while (True):
idx = findClosestCentroid(X, centroids)
centroids = computeMeans(X, idx, CLUSTERS)
cur_cost = computeCost(X, idx, centroids)
if prev_cost == -1 or prev_cost > cur_cost:
prev_cost = cur_cost
centroids_history.append(centroids)
else:
break
cost = computeCost(X, idx, centroids)
costs.append(cost)
if not update:
continue
if final_cost == -1 or final_cost > cost:
if final_cost != -1:
if (final_cost - cost > threshold):
final_cost = cost
final_clusters = CLUSTERS
final_idx = idx
final_centroids_history = centroids_history
else:
update = False
else:
final_cost = cost
final_clusters = CLUSTERS
final_idx = idx
final_centroids_history = centroids_history
# Plot inertia vs K graph
xx = np.arange(1, max_clusters + 1)
plt.plot(xx, costs, "-o")
plt.xlabel('Number of Clusters')
plt.ylabel('Inertia')
plt.title('Elbow Curve')
plt.show()
return [final_clusters, final_idx, final_centroids_history, final_cost]
def gradientDescent(X, y, theta, alpha, num_iters):
#GRADIENTDESCENT Performs gradient descent to learn theta
# theta = GRADIENTDESENT(X, y, theta, alpha, num_iters) updates theta by
# taking num_iters gradient steps with learning rate alpha
# Initialize some useful values
m = len(y) # number of training examples
J_history = zeros((num_iters, 1))
for iteration in range(num_iters):
# ====================== YOUR CODE HERE ======================
# Instructions: Perform a single gradient step on the parameter vector
# theta.
#
# Hint: While debugging, it can be useful to print out the values
# of the cost function (computeCost) and gradient here.
#
# ============================================================
# Save the cost J in every iteration
J_history[iteration] = computeCost(X, y, theta)
#J_history(iter)
return (theta, J_history)
def gradientDescent(x, y, theta, alpha, iters):
m = len(y)
J_history = []
for j in range(iters):
"""
== == == == == == == == == == == YOUR CODE HERE == == == == == == == == == == ==
Instructions: Perform a single gradient step on the parameter vector theta.
Hint: While debugging, it can be useful to print out the values of the cost
function(computeCost) and gradient here.
== == == == == == == == == == == == == == == == == == == == == == == == == == ==
"""
sum0 = 0
sum1 = 0
for i in range(m):
sum0 += hyp(x[i], theta) - y[i]
sum1 += (hyp(x[i], theta) - y[i]) * x[i]
sum0 /= m
sum1 /= m
theta[0] = theta[0] - alpha * sum0
theta[1] = theta[1] - alpha * sum1
J_history.append(computeCost(x, y, theta))
return theta
def gradientDescent(X, y, theta, num_iters: int = 1500, alpha: float = 0.001):
J_history = []
m = X.shape[0]
for num_iter in range(1, num_iters + 1):
slopes = (1 / m) * (np.dot(X.transpose(), (np.dot(X, theta) - y)))
theta = theta - alpha * slopes
cost = computeCost(X, y, theta)
J_history.append(cost)
return [theta, J_history]
def gradientDescent(X, y, theta, alpha, epoch):
temp = np.matrix(np.zeros(theta.shape)) #temp为（1,2）的矩阵，为了同步更新
cost = np.zeros(epoch) #存放每次迭代完成后计算出的cost，迭代epoch次，存放epoch个值
m = X.shape[0]
for i in range(epoch):
temp = theta - (alpha / m) * (X * theta.T - y).T * X
theta = temp
cost[i] = computeCost(X, y, theta)
return theta, cost
def gradientDescent(X, y, theta, alpha, iterations):
m = len(y)
J_history = np.zeros((iterations, 1))
for i in range(iterations):
h = np.dot(X, theta)
# Fix line: output is (97, 2) should be (2, 1)
theta = (theta -
np.transpose(alpha * np.dot(np.transpose(h - y), X)) / m)
J_history[i] = computeCost(X, y, theta)
return theta, J_history
def gradientDescent(X, y, theta, alpha, numIter):
""" Perform Linear Regression using Gradient Descent"""
costHistory = [0]
m = len(X) # Number of training examples
for i in range(numIter):
temp = theta[:] # Copy theta vector to a temporary vector for update
temp = X.T.dot((X.dot(theta) - y)) # Performing the entire operation of grad des using
# matrix multiplication
theta = theta - ((alpha / m) * temp)
costHistory.append(computeCost(X, y, theta, m))
if costHistory[-1] > costHistory[-2]:
print "Cost is not converging. Please check alpha. Exiting"
break
return theta
def gradientDescent(X, y, theta, alpha, numIter):
''' Perform Linear Regression using Gradient Descent'''
costHistory = [0]
m = len(X) # Number of training examples
for i in range(numIter):
temp = theta[:] # Copy theta vector to a temporary vector for update
temp = (X.T.dot((X.dot(theta) - y))
) # Performing the entire operation of grad des using
# matrix multiplication
theta = theta - ((alpha / m) * temp)
costHistory.append(computeCost(X, y, theta, m))
if costHistory[-1] > costHistory[-2]:
print 'Cost is not converging. Please check alpha. Exiting'
break
return theta
def gradientdescent(X,y,theta,alpha, num_iters): m = len(y) J_history = zerovector(num_iters) for iter in range(0, num_iters): ##temp0 = 0 ##temp1 = 0 ##for i in range(0,m): ##temp0 = temp0 + (np.float(( X[i,]*theta) - y[i])) * X[i,0] ##temp1 = temp1 + (np.float(( X[i,]*theta) - y[i])) * X[i,1] ##theta[0,0] = theta[0,0] - alpha / m * temp0 ##theta[1,0] = theta[1,0] - alpha / m * temp1 A = X*theta - y theta[0,0] = theta[0,0] - alpha / m * (X[:,0].transpose()*A) theta[1,0] = theta[1,0] - alpha / m * (X[:,1].transpose()*A) J_history[iter] = computeCost(X, y, theta) print J_history[iter] Plot(range(0,num_iters),J_history) return theta
def gradientDescent(X, y, theta, alpha, num_iters):
# """
# GRADIENTDESCENT Performs gradient descent to learn theta
# theta = GRADIENTDESCENT(X, y, theta, alpha, num_iters) updates theta by
# taking num_iters gradient steps with learning rate alpha
# """
m = len(y)
J_history = np.zeros((num_iters, 1))
for i in range(num_iters):
temp0 = theta[0] - (alpha *
(1 / m) * np.sum(np.subtract(np.dot(X, theta), y)))
temp1 = theta[1] - (alpha * (1 / m) * np.sum(
np.multiply(np.subtract(np.dot(X, theta), y), X[:, 1].reshape(
m, 1))))
theta[0] = temp0
theta[1] = temp1
J_history[i] = computeCost(X, y, theta)
return (theta, J_history)
def gradientDescent(X, y, theta, alpha, num_iters):
'''
Input: X,y,theta and num_iters
Output: updated theta and cost function history
'''
# Initialize some useful values
m = len(y) # number of training examples
J_history = np.zeros((num_iters, 1))
for iter in range(num_iters):
hypothesis = X @ theta
delta = 1 / m * ((hypothesis.transpose() - y) @ X)
theta = theta - (alpha * delta.transpose())
# Save the cost J in every iteration
J_history[iter] = computeCost(X, y, theta)
return (theta, J_history)
# 测试之后发现，X,y由 pandas.dataframe-->numpy.ndarray #代价函数是应该是numpy矩阵，所以我们需要转换X和Y，然后才能使用它们。 我们还需要初始化theta。 #X = np.matrix(X.values) X = X.values #都是可以的 y = np.matrix(y.values) #print(X) #初始化theta=[0,0] ,即我们的终极目标theta1初始化为0，theta2初始化为0 theta = np.matrix([0, 0]) #检查一下X y theta 的维度 #print(X.shape,y.shape,theta.shape) #(97, 2) (97, 1) (1, 2) #计算初始代价函数的值 computeCost(X, y, theta) #print(computeCost(X,y,theta)) #32.072733877455676 值为当(theta0,theta1) = [0,0]时的代价函数的值 #======================= Part 3: Gradient descent========== #初始化：学习速率α和要执行的迭代次数epoch alpha = 0.01 epoch = 1000 #现在让我们运行梯度下降算法来将我们的参数θ适合于训练集 #final_theta是迭代了1000次后得到的最优解【*,*】 final_theta, cost = gradientDescent(X, y, theta, alpha, epoch) #最后，我们可以使用我们拟合的参数计算训练模型的代价函数（误差) #32.072733877455676 值为当(theta0,theta1) = [0,0]时的代价函数的值
import os
os.chdir('/home/morena/MachineLearning/AndrewNg_Python/2.LinearRegression/machine-learning-ex1/ex1')
# Inputs to the functions
X1 = np.transpose(
np.array([np.ones(20), np.exp(1) + np.exp(2) * np.arange(0.1, 2.1, 0.1)]))
Y1 = np.transpose(np.array([X1[:, 1]+np.sin(X1[:, 0])+np.cos(X1[:, 1])]))
X2 = np.transpose(np.array([X1[:, 1]**0.5, X1[:, 1]**0.25]))
X2 = np.concatenate((X1, X2), axis=1)
Y2 = np.array(Y1**0.5 + Y1)
# WarmUpExercise
print('Print WarmUpExercise:\n{}'.format(warmUpExercise(5)))
# computeCost with one variable
print('Print computeCost with one variable:\n{}'.format(computeCost(X1, Y1.transpose(),
np.array([0.5, -0.5]).transpose())))
# gradientDescent with one variable
(theta, J_history) = gradientDescent(
X1, Y1[:, 0], np.array([0.5, -0.5]).transpose(), 0.01, 10)
print('theta_single = {}, J_history_single = {}'.format(theta, J_history))
# Feature Normalization
[X_norm, mu, sigma] = featureNormalize(X2[:, 1:3])
print('X_norm:\n{}\nmu:{}\nsigma{}'.format(X_norm, mu, sigma))
# computeCost with multiple variables
costMulti = computeCost(X2, Y2.transpose(), np.array(
[0.1, 0.2, 0.3, 0.4]).transpose())
print('Print computeCost with multiple variables\n{}'.format(costMulti))
X = np.mat(np.array([onevector(m), vX]))
X = X.transpose()
#X is a the form:
# [ 1 x(1) ]
# [ 1 x(2) ]
# [ 1 x(3) ]
theta = np.mat(np.array([zerovector(2)]))
theta = theta.transpose()
#Theta is of the form:
#[ theta0 ]
#[ theta1 ]
iterations = 15#00
alpha = 0.01
J = computeCost(X, y , theta)
print("Initial cost J(0):")
print J
theta = gradientdescent(X, y, theta, alpha, iterations)
print("Calculated Theta:")
print theta
predict1 = [[1 , 3.5]] * theta
print('For population = 35,000, we predict a profit of:')
print predict1*10000
predict2 = [[1 , 7]] * theta
print('For population = 70,000, we predict a profit of:')
print predict2*10000
theta1_vals = np.linspace(-10,10,25)
y.append(float(y_))
plotData(x=X, y=y)
## ======== Part 3: Cost and Gradient Descent ======== ##
X1 = [[1, X[i]] for i in range(len(X))] # Add a column of 1's to X
theta = [0, 0]
# Some Gradient Descent settings
iterations = 1500
alpha = .01
print('Testing the cost function... \n')
initial_cost = computeCost(X=X1, y=y, m=m, theta=theta)
print('The initial cost (before Gradient Descent optimization) = ',
initial_cost)
print('The expected value (approx) 32.07\n')
another_test = computeCost(X=X1, y=y, m=m, theta=[-1, 2])
print('With theta_0 = -1 and theta_1 = 2 the cost value is ', another_test)
print('The expected value (approx) 54.24\n')
print('Runing Gradient Descent... \n')
thetaGD = gradienDescent(X=X1,
y=y,
m=m,
theta=theta,
iterations=iterations,
pause()
## =================== Part 3: Cost and Gradient descent ===================
X = zeros((m, 2))
X[:, 0] = ones(m) # Add a column of ones to x
X[:, 1] = data[:, 0]
theta = zeros((2, 1)) # initialize fitting parameters
# Some gradient descent settings
iterations = 1500
alpha = 0.01
print('Testing the cost function ...')
# compute and display initial cost refer to computeCost
J = computeCost(X, y, theta)
print('With theta = [0 0]\nCost computed = {0:.2f}'.format(J))
print('Expected cost value (approx) 32.07')
pause()
# further testing of the cost function
J = computeCost(X, y, np.array([[-1], [2]]))
print('With theta = [-1 2]\nCost computed = {0:.2f}'.format(J))
print('Expected cost value (approx) 54.24')
print('Program paused. Press enter to continue.')
print('Running Gradient Descent ...')
# run gradient descent refer to gradientDescent
theta, J = gradientDescent(X, y, theta, alpha, iterations)
# print theta to screen
# Add a column of ones to X (intercept term)
X = np.stack((np.ones(m).reshape(m), X), axis=-1)
# theta in linear regression with one variable represents intercept and slope
# Need one theta for all features in training dataset
(m, n) = X.shape
theta = np.zeros(n).reshape(n, 1) # initialize fitting parameters to zero
# gradient descent settings
iterations = 1500
alpha = 0.01
print('\nTesting the cost function ...\n')
# compute and display initial cost
J = computeCost(X, y, theta)
print("With theta = {}\nCost computed = {}\n".format(theta.transpose(), J))
# further testing of the cost function
J = computeCost(X, y, [[-1], [2]])
print('\nWith theta = [-1 ; 2]\nCost computed = {}\n'.format(J))
print('Program paused. Press enter to continue. \n')
# Calculate gradientDescent
print('\nRunning Gradient Descent ...\n')
(theta, J_history) = gradientDescent(X, y, theta, alpha, iterations)
print('Theta found by gradient descent:\n')
print('{}\n'.format(theta))
# check the cost function trend over iterations (should never increase)
iters = np.arange(1, iterations + 1).reshape(iterations, 1)
raw_input()
## =================== Part 3: Gradient descent ===================
print 'Running Gradient Descent ...'
X = hstack((ones((m, 1)), vstack(data[:,0]))) # Add a column of ones to x
theta = zeros((2, 1)) # initialize fitting parameters
# Some gradient descent settings
iterations = 1500
alpha = 0.01
# compute and display initial cost
computeCost(X, y, theta)
# run gradient descent
(theta, J_history) = gradientDescent(X, y, theta, alpha, iterations)
# print theta to screen
print 'Theta found by gradient descent: '
print '%f %f \n' % (theta[0].var(), theta[1].var())
# Plot the linear fit
#hold on; # keep previous plot visible
plot(vstack(X[:,1]), X.dot(theta), '-')
legend(('Training data', 'Linear regression'))
# not sure how to avoid overlaying any more plots on this figure - call figure()?
# Predict values for population sizes of 35,000 and 70,000
fig = plt.figure()
ax = plt.axes()
plt.scatter(X, y)
# display here as needed to get an idea of the pattern and help decide the ML algorithm to use
# plt.show()
input('Press ENTER to continue...')
# Calculate Cost Function
d = np.ones((m))
X = np.column_stack((d, X))
theta = np.array([0, 0])
# theta = np.zeros((2, 1))
# theta = [-1 , 2]
print('\nTesting the cost function ...\n')
J = computeCost(X, y, theta) # compute and display initial cost
print('J= ', J)
input('Press ENTER to continue...')
# Some gradient descent settings
iterations = 1500
alpha = 0.01
# run gradient descent
print('\nRunning Gradient Descent ...\n')
theta = gradientDescent(X, y, theta, alpha, iterations)
print('Theta = ', theta)
input('Press ENTER to continue...')
# Introducing a variable m which is a number of training examples
m = len(y)
# Plot Data
# Calling a function plotData from plotData.py script
plotData(X, y)
# Script is paused. User should press Enter in order to continue
raw_input('Program paused. Press <Enter> to continue.\n')
# =================== Part 3: Gradient descent ===================
print('Running Gradient Descent ...\n')
# Add a column of ones to x
# Preparing a y variable
X = np.array([np.ones(m), data[:, 1]]).T
y = np.array([data[:, 1]]).T
# initialize fitting parameters
theta = np.zeros((2, 1))
# Some gradient descent settings
iterations = 1500
alpha = 0.01
# Compute and display initial cost
print(computeCost(X, y, theta))
# Run gradient descent
theta = gradientDescent(X, y, theta, alpha, iterations)
