def output(partId):
# Random Test Cases
X = np.stack([
np.ones(20),
np.exp(1) * np.sin(np.arange(1, 21)),
np.exp(0.5) * np.cos(np.arange(1, 21))
],
axis=1)
y = np.reshape((np.sin(X[:, 0] + X[:, 1]) > 0).astype(float), (20, 1))
if partId == '1':
out = formatter('%0.5f ', sigmoid(X))
elif partId == '2':
out = formatter(
'%0.5f ',
costFunction(np.reshape(np.array([0.25, 0.5, -0.5]), (3, 1)), X,
y))
elif partId == '3':
cost, grad = costFunction(
np.reshape(np.array([0.25, 0.5, -0.5]), (3, 1)), X, y)
out = formatter('%0.5f ', grad)
elif partId == '4':
out = formatter(
'%0.5f ',
predict(np.reshape(np.array([0.25, 0.5, -0.5]), (3, 1)), X))
elif partId == '5':
out = formatter(
'%0.5f ',
costFunctionReg(np.reshape(np.array([0.25, 0.5, -0.5]), (3, 1)), X,
y, 0.1))
elif partId == '6':
cost, grad = costFunctionReg(
np.reshape(np.array([0.25, 0.5, -0.5]), (3, 1)), X, y, 0.1)
out = formatter('%0.5f ', grad)
return out
def learningCurve(X, y, Xval, yval, lam):
m = len(y)
error_train = np.zeros([m, 1])
error_val = np.zeros([m, 1])
iterations = np.arange(m)
for i in range(1,m):
error_theta_train = trainLinearReg(X[:i,:],y[:i,:],lam)
error_train[i] = costFunctionReg(error_theta_train[:,np.newaxis],X[:i,:],y[:i,:],0)
error_val[i] = costFunctionReg(error_theta_train[:,np.newaxis],Xval,yval,0)
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.plot(iterations,error_val,'g')
plt.plot(iterations,error_train,'b')
plt.show()
return 1
def output(partId):
# Random Test Cases
X = column_stack((ones(20), exp(1) * sin(arange(1, 21, 1)), exp(0.5) * cos(arange(1, 21, 1))))
y = (sin(X[:,0] + X[:,1]) > 0).astype(int)
if partId == '1':
return sprintf('%0.5f ', sigmoid(X))
elif partId == '2':
return sprintf('%0.5f ', costFunction(array([0.25, 0.5, -0.5]), X, y))
elif partId == '3':
cost, grad = costFunction(array([0.25, 0.5, -0.5]), X, y)
return sprintf('%0.5f ', grad)
elif partId == '4':
return sprintf('%0.5f ', predict(array([0.25, 0.5, -0.5]), X))
elif partId == '5':
return sprintf('%0.5f ', costFunctionReg(array([0.25, 0.5, -0.5]), X, y, 0.1))
elif partId == '6':
cost, grad = costFunctionReg(array([0.25, 0.5, -0.5]), X, y, 0.1)
return sprintf('%0.5f ', grad)
def _calCost(self, trainObj):
print "entering loop-----------------------------"
cost = Decimal('+Infinity')
prob = []
for i in range(0, self.numIter):
costFuncObj = costFunctionReg(trainObj.xList, trainObj.yList,
self.lambdaVal, self.lambdaVal,
self.weightList)
#if(costFuncObj.cost<cost):
cost = costFuncObj.cost
prob = costFuncObj.prob0
self.weightList = costFuncObj.weightList
#else:break;
#print "next iteration----------------------------------------";
return prob
def lrCostFunction(theta, X, y, Lambda):
"""computes the cost of using
theta as the parameter for regularized logistic regression and the
gradient of the cost w.r.t. to the parameters.
"""
# ====================== YOUR CODE HERE ======================
# Instructions: Compute the cost of a particular choice of theta.
# You should set J to the cost.
#
# Hint: The computation of the cost function and gradients can be
# efficiently vectorized. For example, consider the computation
#
# sigmoid(X * theta)
#
# Each row of the resulting matrix will contain the value of the
# prediction for that example. You can make use of this to vectorize
# the cost function and gradient computations.
#
J = 0
# =============================================================
return costFunctionReg(theta, X, y, Lambda)
% regression).
%
'''
# Add Polynomial Features
# Note that mapFeature also adds a column of ones for us, so the intercept term is handled
X = mapFeature(X[:,0], X[:,1])
# Initialize fitting parameters
initial_theta = np.zeros(X.shape[1])
# Set regularization parameter lambda to 1
Lambda = 1.0
# Compute and display initial cost and gradient for regularized logistic regression
cost = costFunctionReg(initial_theta, X, y, Lambda)
print'Cost at initial theta (zeros): {}'.format(cost)
raw_input("Program paused. Press Enter to continue.")
''' ============= Part 2: Regularization and Accuracies =============
% Optional Exercise:
% In this part, you will get to try different values of lambda and
% see how regularization affects the decision coundart
%
% Try the following values of lambda (0, 1, 10, 100).
%
from scipy.optimize import minimize
if __name__ == '__main__':
data2 = np.loadtxt('ex2data2.txt', delimiter=',')
X_original = X = data2[:, :2]
y = data2[:, 2]
pd.plotData(X, y, True, reg=True)
raw_input("Press Enter to create new features")
X = mf.mapFeature(X[:, 0], X[:, 1])
print "New features created"
m, n = X.shape
_lambda = 1.0
initial_theta = np.zeros(n)
cost, grad = cfr.costFunctionReg(initial_theta, X, y, _lambda)
print "Cost calculated for initial theta: %f" % cost
print "Grad calculated for initial theta:\n%s " % grad
raw_input("Press Enter to optimise theta.....")
objective = lambda t: cfr.costFunctionReg(t, X, y, _lambda)[0]
result = minimize(objective,
initial_theta,
method='CG',
options={
'maxiter': 100000,
'disp': False,
})
theta = result.x
print theta
# Add Polynomial Features # Note that mapFeature also adds a column of ones for us, so the intercept # term is handled X = X.apply(mapFeature, axis=1) # Initialize fitting parameters initial_theta = np.zeros(X.shape[1]) # Set regularization parameter lambda to 1 Lambda = 0.0 # Compute and display initial cost and gradient for regularized logistic # regression cost = costFunctionReg(initial_theta, X, y, Lambda) print 'Cost at initial theta (zeros): %f' % cost grad = gradientFunctionReg(initial_theta, X, y, Lambda) #print 'grad at top 5 :%f' % [grad[i] for i in range(5)] print 'grad:', ["%0.4f" % i for i in grad] print 'TEST CASE 1: theta:ones, lambda=10.0 ' initial_theta_ones = np.ones(X.shape[1]) cost = costFunctionReg(initial_theta_ones, X, y, 1.0) print 'Cost at initial theta (ones): %f' % cost
plt.show()
plt.close(fig)
# =========== Part 1: Regularized Logistic Regression ============
m = y.shape[0]
X = mapFeature(X[:, 0, np.newaxis], X[:, 1, np.newaxis], m)
n = X.shape[1]
initial_theta = np.zeros([n])
# Set regularization parameter lambda to 1
reg_factor = 1.0
# Compute and display initial cost and gradient for regularized logistic regression
cost, grad = costFunctionReg(initial_theta,
X,
y,
reg_factor,
requires_grad=True)
print('Cost at initial theta (zeros): %f' % cost)
print('Expected cost (approx): 0.693\n')
print('Gradient at initial theta (zeros) - first five values only:\n',
grad[:5])
print('Expected gradients (approx) - first five values only:')
print(' 0.0085, 0.0188, 0.0001, 0.0503, 0.0115\n')
input("Program paused. Press enter to continue.\n\n")
# Compute and display cost and gradient with all-ones theta and lambda = 10
test_theta = np.ones([n], dtype=float)
cost, grad = costFunctionReg(test_theta,
X,
y,
#
# To do so, you introduce more features to use -- in particular, you add
# polynomial features to our data matrix (similar to polynomial
# regression).
# Note that mapFeature also adds a column of ones for us, so the intercept
# term is handled
X = mapFeature(X[:, [0]], X[:, [1]])
# Initialize fitting parameters
initial_theta = np.zeros(X.shape[1])
# Set regularization parameter lambda to 1
lamda = 1
cost, _ = costFunctionReg(initial_theta, X, y, lamda)
print('Cost at initial theta (zeros):', cost)
# ============= Part 2: Regularization and Accuracies =============
# Optional Exercise:
# In this part, you will get to try different values of lambda and
# see how regularization affects the decision coundart
#
# Try the following values of lambda (0, 1, 10, 100).
#
# How does the decision boundary change when you vary lambda? How does
# the training set accuracy vary?
cost_function = lambda p: costFunctionReg(p, X, y, lamda)[0]
grad_function = lambda p: costFunctionReg(p, X, y, lamda)[1]
def ex2_reg():
# Initialization
os.system("cls" if os.name == "nt" else "clear")
# Load Data
data = loadtxt("ex2data2.txt", delimiter=",")
# The first two columns contains the exam scores and the third column
# contains the label.
X = data[:, 0:2]
y = data[:, 2]
plotData(X, y)
# Labels and Legend
plt.xlabel("Microchip Test 1")
plt.ylabel("Microchip Test 2")
# Specified in plot order
plt.plot([], [], "bo", label="y = 1")
plt.plot([], [], "r*", label="y = 0")
plt.legend()
plt.show()
# =========== Part 1: Regularized Logistic Regression ============
# In this part, you are given a dataset with data points that are not
# linearly separable. However, you would still like to use logistic
# regression to classify the data points.
#
# To do so, you introduce more features to use -- in particular, you add
# polynomial features to our data matrix (similar to polynomial
# regression).
#
# Add Polynomial Features
# Note that mapFeature also adds a column of ones for us, so the intercept
# term is handled
X = mapFeature(X[:, 0], X[:, 1])
# Initialize fitting parameters
initial_theta = zeros(size(X, 1), dtype=float).reshape(size(X, 1), 1)
# Set regularization parameter lambda to 1
_lambda = 1
# Compute and display initial cost and gradient for regularized logistic
# regression
cost, grad = costFunctionReg(initial_theta, X, y, _lambda)
print("Cost at initial theta (zeros): {:0.3f}".format(cost))
print("Expected cost (approx): 0.693")
print("Gradient at initial theta (zeros) - first five values only:")
for g in grad[:5]:
print("{:0.4f}".format(g))
print("Expected gradients (approx) - first five values only:")
print("0.0085\n0.0188\n0.0001\n0.0503\n0.0115")
input("Program paused. Press enter to continue.")
# Compute and display cost and gradient
# with all-ones theta and lambda = 10
test_theta = ones(size(X, 1))
cost, grad = costFunctionReg(test_theta, X, y, 10)
print("Cost at test theta (with lambda = 10): {:0.2f}".format(cost))
print("Expected cost (approx): 3.16\n")
print("Gradient at test theta - first five values only:")
for g in grad[:5]:
print("{:0.4f}".format(g))
print("Expected gradients (approx) - first five values only:")
print("0.3460\n0.1614\n0.1948\n0.2269\n0.0922")
input("Program paused. Press enter to continue.")
# ============= Part 2: Regularization and Accuracies =============
# Optional Exercise:
# In this part, you will get to try different values of lambda and
# see how regularization affects the decision coundart
#
# Try the following values of lambda (0, 1, 10, 100).
#
# How does the decision boundary change when you vary lambda? How does
# the training set accuracy vary?
#
# Initialize fitting parameters
initial_theta = zeros(size(X, 1), dtype=float).reshape(size(X, 1), 1)
# Set regularization parameter lambda to 1 (you should vary this)
_lambda = 1
# Set Options
options = {"maxiter": 1800}
# Optimize
result = minimize(fun=costFunctionReg,
x0=initial_theta,
args=(X, y, _lambda),
jac=True,
method="TNC",
options=options)
theta = result.x
# Plot Boundary
plotDecisionBoundary(theta, X, y)
# Title
plt.title("lambda = {:g}".format(_lambda))
# Labels and Legend
plt.xlabel("Microchip Test 1")
plt.ylabel("Microchip Test 2")
plt.plot([], [], "bo", label="y = 1")
plt.plot([], [], "r*", label="y = 0")
plt.plot([], [], "c-", label="Decision boundary")
plt.legend()
plt.show()
# Compute accuracy on our training set
p = predict(theta, X)
print("Train Accuracy: {:6f}%".format(mean(p == y) * 100))
# Add Polynomial Features
# Note that mapFeature also adds a column of ones for us, so the intercept
# term is handled
X = mapFeature(data_X[:, 0], data_X[:, 1])
# Initialize fitting parameters
initial_theta = np.zeros((X.shape[1], 1))
# Set regularization parameter lambda to 1
lambda_parameter = 1
# Compute and display initial cost and gradient for regularized logistic
# regression
cost, grad = costFunctionReg(initial_theta, X, y, lambda_parameter)
print('Cost at initial theta (zeros): %f\n' % cost)
print('Expected cost (approx): 0.693\n')
print('Gradient at initial theta (zeros) - first five values only:\n')
print(grad[0:5])
print('Expected gradients (approx) - first five values only:\n')
print(' 0.0085\n 0.0188\n 0.0001\n 0.0503\n 0.0115\n')
print('\nProgram paused. Press enter to continue.\n')
# Compute and display cost and gradient
# with all-ones theta and lambda = 10
test_theta = np.ones((X.shape[1],1))
cost, grad = costFunctionReg(test_theta, X, y, 10)
#
# To do so, you introduce more features to use -- in particular, you add
# polynomial features to our data matrix (similar to polynomial
# regression).
# Note that mapFeature also adds a column of ones for us, so the intercept
# term is handled
X = mapFeature(X[:, [0]], X[:, [1]])
# Initialize fitting parameters
initial_theta = np.zeros(X.shape[1])
# Set regularization parameter lambda to 1
lamda = 1
cost, _ = costFunctionReg(initial_theta, X, y, lamda)
print('Cost at initial theta (zeros):', cost)
# ============= Part 2: Regularization and Accuracies =============
# Optional Exercise:
# In this part, you will get to try different values of lambda and
# see how regularization affects the decision coundart
#
# Try the following values of lambda (0, 1, 10, 100).
#
# How does the decision boundary change when you vary lambda? How does
# the training set accuracy vary?
cost_function = lambda p: costFunctionReg(p, X, y, lamda)[0]
grad_function = lambda p: costFunctionReg(p, X, y, lamda)[1]
theta = fmin_bfgs(cost_function, initial_theta, fprime=grad_function)
# Note that mapFeature also adds a column of ones for us, so the intercept
# term is handled
X = mapFeature(X[:,0], X[:,1])
# Initializae fitting parameters
initial_theta = np.zeros((X.shape[1],))
# Set regularization parameter to 1
lambdaa = 1
# Compute and display initial cost and gradient for regularized logistic
# regression
cost,grad = costFunctionReg(initial_theta, X, y, lambdaa)
print('Cost at initial theta (zeros): %f\n' %cost)
print('Expected cost (approx): 0.693\n')
print('Gradient at initial theta (zeros) - first five values only:\n')
print(grad[0:5])
print('Expected gradients (approx) - first five values only:\n')
print(' 0.0085\n 0.0188\n 0.0001\n 0.0503\n 0.0115\n')
print('\nProgram paused. Press enter to continue.\n')
time.sleep(2)
# Compute and display cost and gradient
# with all-ones theta and lambda = 10
test_theta = np.ones((X.shape[1],))
displayData(np.copy(X), image_size=(20, 20), disp_rows=10, disp_cols=10, padding=1, shuffle=True)
y = y.reshape(-1, )
y[np.where(y == 10)] = 0
m = len(y)
X = np.concatenate((np.ones([m, 1]), X), axis=1)
input("Program paused. Press enter to continue.\n")
# ============ Part 2a: Vectorize Logistic Regression ============
print('===========Testing CostFunctionReg with regularization===========')
test_theta = np.array([-2, -1, 1, 2])
X_test = np.concatenate((np.ones([5, 1]), np.arange(1, 16, dtype=float).reshape(5, 3, order='F') / 10), axis=1)
y_test = np.array([1, 0, 1, 0, 1])
reg_factor_test = 3
J, grad = costFunctionReg(test_theta, X_test, y_test, reg_factor_test, requires_grad=True)
print('Cost: %f' % J)
print('Expected cost: 2.534819\n')
print('Gradients:\n', grad)
print('Expected gradients:\n0.146561\n -0.548558\n 0.724722\n 1.398003\n')
input("Program paused. Press enter to continue.\n")
# ============ Part 2b: One-vs-All Training ============
print('===========Training One-vs-All Logistic Regression...===========\n')
reg_factor = 0.1
num_labels = 10
all_theta = oneVsAll(X, y, num_labels, reg_factor)
input("Program paused. Press enter to continue.\n")
# ================ Part 3: Predict for One-Vs-All ================
#
# % Add Polynomial Features
#
# % Note that mapFeature also adds a column of ones for us, so the intercept
# % term is handled
X = MP.mapFeature(X[:, 0], X[:, 1])
#
# % Initialize fitting parameters
initial_theta = np.zeros(X.shape[1])
#
# % Set regularization parameter lambda to 1
Lambda = 1
#
# % Compute and display initial cost and gradient for regularized logistic
# % regression
[cost, grad] = CFR.costFunctionReg(initial_theta, X, y, Lambda)
print('Cost at initial theta (zeros):', cost)
print('Expected cost (approx): 0.693\n')
print('Gradient at initial theta (zeros) - first five values only:', grad[:5])
print('Expected gradients (approx) - first five values only:')
print(' 0.0085\n 0.0188\n 0.0001\n 0.0503\n 0.0115\n')
input("Press Enter to continue...")
# % Compute and display cost and gradient
# % with all-ones theta and lambda = 10
test_theta = np.ones([X.shape[1], 1])
[cost, grad] = CFR.costFunctionReg(test_theta, X, y, 10)
print('\nCost at test theta (with lambda = 10):', cost)
print('Expected cost (approx): 3.16\n')
plot.title('Scatter Plot of Training Data')
plot.xlabel('Microchip Test 1')
plot.ylabel('Microchip Test 2')
pld.PlotData(X, y)
# Feature Mapping.
X = mf.mapFeature(X[:, 0], X[:, 1])
m, n = X.shape
theta = np.zeros(n)
lambdaa = 1
J, grad = cfr.costFunctionReg(theta, X, y, lambdaa)
print('Cost at initial theta (zeros): ', J[0])
print('Expected cost (approx): 0.693')
print('Gradient at initial theta (zeros) - first five values only: ', grad[:,0:5]);
print('Expected gradients (approx) - first five values only: ');
print(' 0.0085\n 0.0188\n 0.0001\n 0.0503\n 0.0115\n');
###
# Tryinmg different values of lambda (0, 1, 10, 100).
###
# https://docs.scipy.org/doc/scipy/reference/optimize.minimize-cg.html
wrapping =lambda t : cfr.costFunctionReg(t, X, y, lambdaa)[0]
result = minimize( wrapping, theta, method='CG', options={ 'maxiter': 400, 'disp': False})
# Add Polynomial Features
# Note that mapFeature also adds a column of ones for us, so the intercept
# term is handled
X = mapFeature(X[:, 0], X[:, 1])
m, n = X.shape
# Initialize fitting parameters
init_theta = np.zeros((n, 1))
# Set regularization parameter lambda to 1
lambda_reg = 1
# Compute and display initial cost and gradient for regularized logistic regression
cost = costFunctionReg(init_theta, X, y, lambda_reg)
print('Cost at initial theta (zeros): {:f}'.format(cost))
raw_input('\nProgram paused. Press enter to continue.\n')
## ============= Part 2: Regularization and Accuracies =============
# Optional Exercise:
# In this part, you will get to try different values of lambda and
# see how regularization affects the decision coundart
#
# Try the following values of lambda (0, 1, 10, 100).
#
# How does the decision boundary change when you vary lambda? How does
# the training set accuracy vary?
print('Plotting data with + indicating (y = 1) examples and o indicating (y = 0) examples.')
f1 = plotDecisionBoundary.plotData2(X, y)
plt.show()
## =========== Part 1: Regularized Logistic Regression ============
# In this part, you are given a dataset with data points that are not linearly separable.
# However, you would still like to use logistic regression to classify the data points.
#
# To do so, you introduce more features to use -- in particular, you add polynomial features
# to our data matrix (similar to polynomial regression).
X = mapFeature.mapFeature(X[:,0], X[:,1])
initial_theta = (np.zeros(X.shape[1])).reshape(X.shape[1],1)
xlambda = 1
cost, grad = costFunctionReg.costFunctionReg(initial_theta, X, y, xlambda)
print('Cost at initial theta (zeros): %.3f\nExpected cost (approx): 0.693\nGradient at initial theta (zeros) - first five values only:'%cost)
print(grad[0:5])
print('Expected gradients (approx) - first five values only:\n0.0085\n 0.0188\n 0.0001\n 0.0503\n 0.0115\n')
input('Program paused. Press enter to continue.')
## ============= Part 2: Regularization and Accuracies =============
# Optional Exercise:
# In this part, you will get to try different values of lambda and see how regularization affects the decision coundart
# Try the following values of lambda (0, 1, 10, 100).
initial_theta = (np.zeros(X.shape[1])).reshape(X.shape[1],1)
xlambda = 1
# Optimize
result = opt.minimize(fun=costFunctionReg.costFunctionReg,x0=initial_theta,args=(X,y,xlambda),method='TNC',jac='Gradient',callback=callable)
theta = result.x
# Add Polynomial Features
# Note that mapFeature also adds a column of ones for us, so the intercept
# term is handled
X = mf.mapFeature(X[:,0], X[:,1])
m,n = X.shape
# Initialize fitting parameters
initial_theta = np.zeros((n, 1))
# Set regularization parameter lambda to 1
lambda_reg = 0.1
# Compute and display initial cost
# gradient is too large to display in this exercise
cost = cfr.costFunctionReg(initial_theta, X, y, lambda_reg)
print('Cost at initial theta (zeros): {:f}'.format(cost))
# print('Gradient at initial theta (zeros):')
# print(grad)
raw_input('Program paused. Press enter to continue.\n')
## ============= Part 2: Regularization and Accuracies =============
# Optional Exercise:
# In this part, you will get to try different values of lambda and
# see how regularization affects the decision coundart
#
# Try the following values of lambda (0, 1, 10, 100).
#
def ex2_reg():
# Machine Learning Online Class - Exercise 2: Logistic Regression
#
# Instructions
# ------------
#
# This file contains code that helps you get started on the second part
# of the exercise which covers regularization with logistic regression.
#
# You will need to complete the following functions in this exericse:
#
# sigmoid.m
# costFunction.m
# predict.m
# costFunctionReg.m
#
# For this exercise, you will not need to change any code in this file,
# or any other files other than those mentioned above.
#
# Initialization
#clear ; close all; clc
# Load Data
# The first two columns contains the X values and the third column
# contains the label (y).
data = np.loadtxt('ex2data2.txt', delimiter=',')
X = np.reshape(data[:, 0:2], (data.shape[0], 2))
y = np.reshape(data[:, 2], (data.shape[0], 1))
pos_handle, neg_handle = plotData(X, y, ['y = 1', 'y = 0'])
# Put some labels
#hold on;
# Labels and Legend
plt.xlabel('Microchip Test 1')
plt.ylabel('Microchip Test 2')
# Specified in plot order
plt.legend(handles=[pos_handle, neg_handle])
plt.savefig('figure1.reg.png')
# =========== Part 1: Regularized Logistic Regression ============
# In this part, you are given a dataset with data points that are not
# linearly separable. However, you would still like to use logistic
# regression to classify the data points.
#
# To do so, you introduce more features to use -- in particular, you add
# polynomial features to our data matrix (similar to polynomial
# regression).
#
# Add Polynomial Features
# Note that mapFeature also adds a column of ones for us, so the intercept
# term is handled
X = mapFeature(X[:, 0], X[:, 1])
# Initialize fitting parameters
initial_theta = np.zeros((X.shape[1], 1))
# Set regularization parameter lambda to 1
lambda_value = 1
# Compute and display initial cost and gradient for regularized logistic
# regression
cost, grad = costFunctionReg(initial_theta, X, y, lambda_value)
print('Cost at initial theta (zeros): %f' % cost)
print('Program paused. Press enter to continue.')
#input()
# ============= Part 2: Regularization and Accuracies =============
# Optional Exercise:
# In this part, you will get to try different values of lambda and
# see how regularization affects the decision coundart
#
# Try the following values of lambda (0, 1, 10, 100).
#
# How does the decision boundary change when you vary lambda? How does
# the training set accuracy vary?
#
# Initialize fitting parameters
initial_theta = np.zeros((X.shape[1], 1))
# Set regularization parameter lambda to 1 (you should vary this)
lambda_value = 1
# Set Options
#options = optimset('GradObj', 'on', 'MaxIter', 400);
# Optimize
result = optimize.minimize(costRegOnly,
initial_theta,
args=(X, y, lambda_value),
method=None,
jac=gradientRegOnly,
options={"maxiter": 400})
theta = result.x
# Plot Boundary
pos_handle, neg_handle, boundary_handle = plotDecisionBoundary(
theta, X, y, ['y = 1', 'y = 0'])
plt.title('lambda = %g' % lambda_value)
# Labels and Legend
plt.xlabel('Microchip Test 1')
plt.ylabel('Microchip Test 2')
plt.legend(handles=[pos_handle, neg_handle, boundary_handle])
plt.savefig('figure2.reg.png')
# Compute accuracy on our training set
p = predict(theta, X)
print('Train Accuracy: %f' % (np.mean(
(p == y.flatten()).astype(int)) * 100))
# Add Polynomial Features # Note that mapFeature also adds a column of ones for us, so the intercept # term is handled X = X.apply(mapFeature, axis=1) # Initialize fitting parameters initial_theta = np.zeros(X.shape[1]) # Set regularization parameter lambda to 1 Lambda = 0.0 # Compute and display initial cost and gradient for regularized logistic # regression cost = costFunctionReg(initial_theta, X, y, Lambda) print 'Cost at initial theta (zeros): %f' % cost # ============= Part 2: Regularization and Accuracies ============= # Optimize and plot boundary Lambda = 1.0 result = optimize(Lambda) theta = result.x cost = result.fun # Print to screen print 'lambda = ' + str(Lambda) print 'Cost at theta found by scipy: %f' % cost
import pandas as pd
import numpy as np
from mapFeature import mapFeature
from costFunctionReg import costFunctionReg
from gradRegLog import gradRegLog
import scipy.optimize as opt
import matplotlib.pyplot as plt
dataset = pd.read_csv("ex2data2.txt",
header=None,
names=['teste1', 'teste2', 'resultado'])
dataset.insert(0, 'Ones', 1)
dataset = np.array(dataset)
cols = dataset.shape[1]
X = dataset[:, 0:cols - 1]
y = dataset[:, cols - 1:]
new_x = mapFeature(X)
# print(new_x)
theta = np.zeros(len(new_x.T))
print(theta)
custo = costFunctionReg(theta, new_x, y, 1)
print(custo)
# opt_theta = opt.fmin_tnc(func=costFunctionReg, x0=theta,
# fprime=gradRegLog, args=(new_x, y, 1))
opt_theta = gradRegLog(theta, new_x, y, 1)
print(np.array(opt_theta))
def objectiveFunction(self, var): return costFunctionReg.costFunctionReg(var, self.X, self.y, self.L)
from costFunctionReg import costFunctionReg, gradientReg
data2 = loaddata('ex2data2.txt', ',')
y = np.c_[data2[:, 2]]
X = data2[:, 0:2]
''' Since we see that the data cannot be separated linearly, we shall
create more features to let the logistic regressor design a more complex boundary which will fit the data.
We use a technique called feature mapping as shown below. However feature mapping makes the data prone to overfitting. '''
poly = PolynomialFeatures(6)
XX = poly.fit_transform(
data2[:, 0:2]
) # create more features from data as terms of X1 and X2 upto the sixth power, called feature mapping, helps us fit the data better
initial_theta = np.zeros(XX.shape[1])
print(costFunctionReg(initial_theta, 1, XX, y))
fig, axes = plt.subplots(1, 3, sharey=True, figsize=(17, 5))
# Decision boundaries
# Lambda = 0 : No regularization --> too flexible, overfitting the training data
# Lambda = 1 : Looks about right
# Lambda = 100 : Too much regularization --> high bias
for i, C in enumerate([0, 1, 100]):
# Optimize costFunctionReg
res2 = minimize(costFunctionReg,
initial_theta,
args=(C, XX, y),
method=None,
jac=gradientReg,
options={'maxiter': 3000})
# Add Polynomial Features
# Note that mapFeature also adds a column of ones for us, so the intercept
# term is handled
X = mapFeature(X[:, 0], X[:, 1])
# Initialize fitting parameters
initial_theta = np.zeros((X.shape[1], 1))
# Set regularization parameter lambda to 1
Lambda = 1.0
# Compute and display initial cost and gradient for regularized logistic
# regression
cost = costFunctionReg(initial_theta, X, y, Lambda)
grad = gradientFunctionReg(initial_theta, X, y, Lambda)
print('\n -------------------------- \n')
print('Cost at initial theta (zeros): %f' % cost)
print('Expected cost (approx): 0.693\n')
print('Gradient at initial theta (zeros) - first five values only: ' +
str(grad[0:5]))
print('Expected gradients (approx) - first five values only:\n')
print(' 8.5e-03 1.88e-02 7.7e-05 5.03e-02 1.15e-02\n')
# Compute and display cost and gradient
# with all-ones theta and lambda = 10
test_theta = np.ones((X.shape[1], 1)) # np.atleast_2d(X.shape)
Lambda = 10.0
# Add Polynomial Features
# Note that mapFeature also adds a column of ones for us, so the intercept
# term is handled
X = X.apply(mapFeature, axis=1)
# Initialize fitting parameters
theta = np.zeros(X.shape[1])
# Set regularization parameter lambda to 1
mylambda = 0.0
# Compute and display initial cost and gradient for regularized logistic
# regression
cost = costFunctionReg(theta, X, y, mylambda)
print('Cost at initial theta (zeros): %f' % cost)
# ============= Part 2: Regularization and Accuracies =============
# Optimize and plot boundary
mylambda = 1.0
result = optimize(mylambda)
theta = result.x
cost = result.fun
# Print to screen
print('lambda = ' + str(mylambda))
print('Cost at theta found by scipy: %f' % cost)
# Add Polynomial Features
# Note that mapFeature also adds a column of ones for us, so the intercept
# term is handled
X = mapFeature(X[:, 0], X[:, 1])
# Initialize fitting parameters
initial_theta = np.zeros(X.shape[1], )
# Set regularization parameter lambda to 1
lmbda = 1
# Compute and display initial cost and gradient for regularized logistic
# regression
cost, grad = costFunctionReg(initial_theta, X, y, lmbda)
print('Cost at initial theta (zeros): %f' % cost)
input('Program paused. Press enter to continue.\n')
# ============= Part 2: Regularization and Accuracies =============
# Optional Exercise:
# In this part, you will get to try different values of lambda and
# see how regularization affects the decision coundart
#
# Try the following values of lambda (0, 1, 10, 100).
#
# How does the decision boundary change when you vary lambda? How does
# the training set accuracy vary?
#
# Add Polynomial Features
# Note that mapFeature also adds a column of ones for us, so the intercept
# term is handled
X = mapFeature(X[:, 0], X[:, 1])
# Initialize fitting parameters
initial_theta = np.zeros(X.shape[1])
# Set regularization parameter lambda to 1
lambda_ = 1
# Compute and display initial cost and gradient for regularized logistic
# regression
cost, grad = costFunctionReg(initial_theta, X, y, lambda_)
np.set_printoptions(precision=4, suppress=True)
print('Cost at initial theta (zeros): {:.3f}'.format(cost))
print('Expected cost (approx): 0.693')
print('Gradient at initial theta (zeros) - first five values only:')
print(' {} '.format(grad[:5]))
print('Expected gradients (approx) - first five values only:')
print(' [0.0085 0.0188 0.0001 0.0503 0.0115]')
input('\nProgram paused. Press enter to continue.\n')
# Compute and display cost and gradient
# with all-ones theta and lambda = 10
test_theta = np.ones(X.shape[1])
cost, grad = costFunctionReg(test_theta, X, y, 10)
X = data[:,:2]
y = data[:,2:3]
m,n = X.shape
plt, pos, neg = plotData (X,y)
plt.legend((pos, neg), ('y=1', 'y=0'), loc='upper right')
plt.xlabel('Microchip Test 1')
plt.ylabel('Microchip Test 2')
plt.show()
X = mapFeature(X[:,0].reshape(m,1), X[:,1].reshape(m,1))
print("Mapped X=", X.shape)
m,n = X.shape
initial_theta = np.zeros((n, 1))
_lambda = 1
cost, grad = costFunctionReg(initial_theta, X, y, _lambda, return_grad=True)
print("Cost={}, Gradient={}".format(cost,grad[:5]))
test_theta = np.ones((n, 1))
cost, grad = costFunctionReg(test_theta, X, y, 10, return_grad=True)
print("Cost={}, Gradient={}".format(cost,grad[:5]))
fargs=(X, y, _lambda)
theta = fmin_bfgs(costFunctionReg, x0=initial_theta, args=fargs)
#print(theta)
plotDecisionBoundary(theta, X, y)
plt.title("lambda = "+str(_lambda))
plt.xlabel('Microchip Test 1')
plt.ylabel('Microchip Test 2')
plt.show()
print("Train Accuracy: ",np.mean(predict(theta, X) == y)*100)
from scipy.optimize import fmin_bfgs
from plotData import plotData
from plotDecisionBoundary import plotDecisionBoundary
from mapFeature import mapFeature
from costFunctionReg import costFunctionReg
from predict import predict
data = np.loadtxt("ex2data2.txt", usecols=(0,1,2), delimiter=',',dtype=None)
X = data[:, 0:2]
y = data[:, 2]
y = y[:, np.newaxis]
l = 1
m, n = X.shape
plotData(X, y)
X = mapFeature(X[:, 0][np.newaxis].T, X[:, 1][np.newaxis].T)
m, n = X.shape
theta = np.zeros((1, n))
#find out why is there so huge difference between fmin and fmin_bfgs ?
#fmin gives totaly wrong result
options = {'full_output': True, 'retall': True}
theta, cost, _, _, _, _, _, allvecs = fmin_bfgs(lambda t: costFunctionReg(X, y, t, l), theta, maxiter=400, **options)
plotDecisionBoundary(X, y, theta)
#plt.show()
p = predict(X, theta[np.newaxis])[np.newaxis]
print np.mean((p.T == y)) * 100
# =========== 2.2 Feature mapping ============
# Add Polynomial Features
# Note that map_feature also adds a column of ones for us, so the intercept term is handled
X = mapFeature(X[:, 0], X[:, 1])
m, n = X.shape
# Initialize fitting parameters
initial_theta = np.zeros(n)
# Set regularization parameter lambda to 1
l = 1.0
# ============ 2.3 Cost function and gradient ==========
cost, _ = costFunctionReg(initial_theta, X, y, l)
print('Cost at initial theta (zeros):', cost)
# ============= 2.3.1 Learning parameters using fminunc =============
# Initialize fitting parameters
initial_theta = np.zeros(n)
# Set regularization parameter lambda to t1 (you should vary this)
l = 1.0
# Optimize
theta, nfeval, rc = opt.fmin_tnc(func=costFunctionReg,
x0=initial_theta,
args=(X, y, l),
messages=0)
#print(X.shape , Xtest.shape , Xval.shape)
#1.1
plt.xlabel('Change in water level')
plt.ylabel('Water flowing out of the dam')
plt.plot(X,y,'rx')
#plt.show()
#1.2,1.3
m = len(y)
ones = np.ones((m,1))
X = np.hstack((ones, X))
lam = 0
theta = np.ones([2,1])
J = costFunctionReg(theta,X,y,lam)
print(J)
grad = gradientReg(theta,X,y,lam)
print(grad)
#1.4
lam = 0
thetaOpt = trainLinearReg(X,y,lam)
print(thetaOpt)
hypothesis = np.dot(X,thetaOpt)
#plt.plot(X[:,1],hypothesis,'b')
#plt.show()
# Add Polynomial Features
# Note that mapFeature also adds a column of ones for us, so the intercept
# term is handled
X = mapFeature.mapFeature(X[:, 0:1], X[:, 1:2])
# Initialize fitting parameters
initial_theta = np.zeros((X.shape[1], 1))
# Set regularization parameter lambdaa to 1
lambdaa = 1
# Compute and display initial cost and gradient for regularized logistic
# regression
cost = costFunctionReg.costFunctionReg(initial_theta, X, y, lambdaa)
grad = gradfunReg.gradfunReg(initial_theta, X, y, lambdaa)
grad = np.matrix(grad)
print('Cost at initial theta (zeros):', cost)
print('Expected cost (approx): 0.693\n')
print('Gradient at initial theta (zeros) - first five values only:\n')
print(' \n', grad[0:5, 0])
print('Expected gradients (approx) - first five values only:\n')
print(' 0.0085\n 0.0188\n 0.0001\n 0.0503\n 0.0115\n')
# Compute and display cost and gradient
# with all-ones theta and lambdaa = 10
test_theta = np.ones((X.shape[1], 1))
cost = costFunctionReg.costFunctionReg(test_theta, X, y, 10)
grad = gradfunReg.gradfunReg(test_theta, X, y, 10)
grad = np.matrix(grad)
# Add Polynomial Features
# Note that mapFeature also adds a column of ones for us, so the intercept
# term is handled
X = mapFeature(X[:,0], X[:,1])
# Initialize fitting parameters
initial_theta = zeros(size(X, 1))
# Set regularization parameter lambda to 1
lambda_ = 1.
# Compute and display initial cost and gradient for regularized logistic
# regression
cost, grad = costFunctionReg(initial_theta, X, y, lambda_)
print('Cost at initial theta (zeros): %f' % cost)
print('\nProgram paused. Press enter to continue.')
input()
## ============= Part 2: Regularization and Accuracies =============
# Optional Exercise:
# In this part, you will get to try different values of lambda and
# see how regularization affects the decision coundart
#
# Try the following values of lambda (0, 1, 10, 100).
#
# How does the decision boundary change when you vary lambda? How does
"""
# Note that mapFeature also adds a column of ones for us, so the intercept term is handled
X = mF.mapFeature(X[:,0], X[:,1]);
#Initialize fitting parameters
m,n=np.shape(X)
#initial guess
initial_theta =np.zeros((n, 1),dtype=float);
#regularization parameter .this can be changed
labda = 1.;
#importing optimization module and costfunction with regularization lambda
from scipy import optimize;
import costFunctionReg as cFR
#getting the cost from the function
cost=cFR.costFunctionReg(initial_theta,X,y,labda)
#optimizing using BFGS important to set jac=False i.e. jacobian is set to false
res = optimize.minimize(cFR.costFunctionReg, initial_theta, args=(X,y,labda), \
method='BFGS',jac=False, options={'maxiter':400})
#theta and cost obtained from the cost function
theta=res.x
cost=res.fun
#plotting the decision boundary
pDB.plotDecisionBoundary(theta, X, y)
#adding some plotting elements
pl.xlabel('Microchip Test 1')
pl.ylabel('Microchip Test 2')
pl.title("lambda is %s "%labda)
pl.legend("10")
#predicting on our training set
