def gradient_descent(theta, x_data, y_data):
iters = 200000
for i in range(iters):
grad = costFunction.gradient(theta, x_data, y_data) # grad is a vector
grad.reshape((-1, 1))
alpha = 1e-3
theta -= alpha * grad
if (i + 1) % 1000 == 0:
print("with {} iterations,cost is {}".format(
i + 1, costFunction.cost_function(theta, x_data, y_data)))
return theta.flatten()
X = np.c_[np.ones(
(data.shape[0], 1)
), data[:, 0:
2]] # adds column of ones (first argument) to X data (second argument) AND creates X from data at the same time
y = np.c_[data[:, 2]] # creates Y from data
plotData(data, 'Exam 1 score', 'Exam 2 score', 'Admitted',
'Not admitted') #plots the data
plt.show()
input(
"proceed only if you've completed completed ALL the files, else press Ctrl+C then enter to exit the program"
)
from costFunction import costFunction, gradient
initial_theta = np.zeros(X.shape[1])
cost = costFunction(initial_theta, X, y)
grad = gradient(initial_theta, X, y)
print('Cost: \n', cost)
print('Grad: \n', grad)
res = minimize(
costFunction, initial_theta, args=(X, y), jac=gradient
) # using sklearn's minimize function to optimize our constfunction for theta while taking in X,y and using gradient function
updated_theta = res.x
print(updated_theta) #printing updated theta to the screen
from sigmoid import sigmoid
plotData(data, 'Exam 1 score', 'Exam 2 score', 'Admitted', 'Not admitted')
x1_min, x1_max = X[:, 1].min(), X[:, 1].max(
), #taking min and max of marks in exam 1
x2_min, x2_max = X[:, 2].min(), X[:, 2].max(
), # taking min and max of marks in exam 2
X = np.c_[np.ones((data.shape[0], 1)), data[:, :2]] y = np.c_[data[:, 2]] m = len(y) sns.scatterplot(X[:, 0], X[:, 1], y[:, 0], style=y[:, 0]) #plt.show() ### Part 2: Compute Cost and Gradient import sigmoid g = sigmoid.sigmoid(X) theta = np.zeros((X.shape[1], 1)) from costFunction import costFunction, gradient J = costFunction(X, y, theta) grad = gradient(X, y, theta) ### Part 3: Optimizing using Scipy ''' #optimized = op.minimize(fun=costFunction, x0=theta, args=(X, y), method='TNC', jac=gradient) #op.fmin_bfgs(J, initial_theta) ## scipy doesn't play nice with numpy.ndarray object types? Convert somehow? Avoid numpy altogether? #ValueError: shapes (2,) and (100,1) not aligned: 2 (dim 0) != 100 (dim 0) It's not recognizing theta as a 2x1 matrix for some reason. Even after transposing and moving things around within costFunction ''' print(J) print(grad) # Let's hardcode the values for optimized_theta that we were supposed to get from the minimization function optimized_theta = np.array([-25.16133593, 0.20623171, 0.20147164]) print(optimized_theta)
line_tempt = line_list[i].split(",")
x_data[i, :] = line_tempt[:feature_num]
y_data[i, 0] = line_tempt[-1]
# ========================= Load end =============================
# ========================= plotData =============================
plotData.plot_data(x_data, y_data)
# ========================= compute cost and gradient ===========
# prepare x_data and initiate theta
n = x_data.shape[1]
x_data = np.column_stack((np.ones((m, 1)), x_data))
initial_theta = np.zeros((n + 1, 1)).flatten() # must be a vector
# cost = costFunction.cost_function(initial_theta, x_data, y_data)
# print("initial_theta cost is {} (approx)".format(cost))
# print("expected cost is 0.693")
grad = costFunction.gradient(initial_theta.flatten(), x_data, y_data)
print("initial_theta grad is {}".format(
grad)) #grad can be gotten from costFunction.gradient
print('Expected gradients (approx):\n -0.1000\n -12.0092\n -11.2628\n')
# ============= Part 3: Optimizing using fminunc(matlab)/scipy(python) =============
# theta must be a vector
result = op.minimize(costFunction.cost_function,
initial_theta.flatten(),
args=(x_data, y_data),
method="TNC",
jac=costFunction.gradient)
final_theta = result.x
plotDecisionBoundary.plot_decesion_boundary(final_theta, x_data, y_data)
# ============= optional:gradientDescent ============================================
# try the gradientDescent method to optimize theta
# final_theta = gradientDescent.gradient_descent(initial_theta, x_data, y_data)
#Ysk = Y.replace(0, 2)
Xnp = np.array(X)
Ynp = np.array(Y)
iters = 50000 # Doesnt saturate after 200,000 itarations also
alpha = 0.00001 # 0.00001 upper limit
theta = np.zeros(X.shape[1] + 1) # Initialize with zeroes
cost_change = []
print("Initial theta:", theta)
skmod = LogisticRegression().fit(X, Y)
# Learning
for i in range(iters):
gradient_vector = cf.gradient(theta, Xnp, Ynp)
theta = theta - alpha * gradient_vector
cost_change.append(cf.cost(theta, Xnp, Ynp))
print("Cost after training:", cf.cost(theta, Xnp, Ynp))
print("Thetas after training:", theta)
x_test = np.array([1, 45, 85])
print("Final prediction:", sg.sigmoid(x_test.T.dot(theta)))
input = np.array([[45, 85]])
input.reshape(1, -1)
print("Final Prediciton SK:", skmod.predict(input))
# Assigning colors to students based on acceptance status
colors = []
for i in range(len(Y)):
if (Y[i] == 1):
from sigmoid import *
data = pd.read_csv('ex2data1.txt', header=None, names=['Exam 1 score', 'Exam 2 score', 'Admission'])
# ===================== Part 1: Plotting =====================
plot_data(data)
# ===================== Part 2: Compute Cost and Gradient =====================
data.insert(0, 'ones', 1)
X = data.values[:,:-1]
y = data.values[:,-1]
theta = np.zeros(X.shape[1])
cost = cf.cost_function(theta, X, y)
grad = cf.gradient(theta, X, y)
# ===================== Part 3: Optimizing parameters theta using advanced algorithm =====================
import scipy.optimize as opt
result = opt.fmin_tnc(func=cf.cost_function, x0=theta, fprime=cf.gradient, args=(X, y))
theta = result[0]
# ===================== Part 4: Predict and Accuracies =====================
predict = sigmoid(np.dot(np.array([1, 45, 85]), theta))
def predict(theta, X):
probability = sigmoid(X@theta)
return [1 if x >= 0.5 else 0 for x in probability] # return a list
