def gradient_descent(theta, x_data, y_data): iters = 200000 for i in range(iters): grad = costFunction.gradient(theta, x_data, y_data) # grad is a vector grad.reshape((-1, 1)) alpha = 1e-3 theta -= alpha * grad if (i + 1) % 1000 == 0: print("with {} iterations,cost is {}".format( i + 1, costFunction.cost_function(theta, x_data, y_data))) return theta.flatten()
X = np.c_[np.ones( (data.shape[0], 1) ), data[:, 0: 2]] # adds column of ones (first argument) to X data (second argument) AND creates X from data at the same time y = np.c_[data[:, 2]] # creates Y from data plotData(data, 'Exam 1 score', 'Exam 2 score', 'Admitted', 'Not admitted') #plots the data plt.show() input( "proceed only if you've completed completed ALL the files, else press Ctrl+C then enter to exit the program" ) from costFunction import costFunction, gradient initial_theta = np.zeros(X.shape[1]) cost = costFunction(initial_theta, X, y) grad = gradient(initial_theta, X, y) print('Cost: \n', cost) print('Grad: \n', grad) res = minimize( costFunction, initial_theta, args=(X, y), jac=gradient ) # using sklearn's minimize function to optimize our constfunction for theta while taking in X,y and using gradient function updated_theta = res.x print(updated_theta) #printing updated theta to the screen from sigmoid import sigmoid plotData(data, 'Exam 1 score', 'Exam 2 score', 'Admitted', 'Not admitted') x1_min, x1_max = X[:, 1].min(), X[:, 1].max( ), #taking min and max of marks in exam 1 x2_min, x2_max = X[:, 2].min(), X[:, 2].max( ), # taking min and max of marks in exam 2
X = np.c_[np.ones((data.shape[0], 1)), data[:, :2]] y = np.c_[data[:, 2]] m = len(y) sns.scatterplot(X[:, 0], X[:, 1], y[:, 0], style=y[:, 0]) #plt.show() ### Part 2: Compute Cost and Gradient import sigmoid g = sigmoid.sigmoid(X) theta = np.zeros((X.shape[1], 1)) from costFunction import costFunction, gradient J = costFunction(X, y, theta) grad = gradient(X, y, theta) ### Part 3: Optimizing using Scipy ''' #optimized = op.minimize(fun=costFunction, x0=theta, args=(X, y), method='TNC', jac=gradient) #op.fmin_bfgs(J, initial_theta) ## scipy doesn't play nice with numpy.ndarray object types? Convert somehow? Avoid numpy altogether? #ValueError: shapes (2,) and (100,1) not aligned: 2 (dim 0) != 100 (dim 0) It's not recognizing theta as a 2x1 matrix for some reason. Even after transposing and moving things around within costFunction ''' print(J) print(grad) # Let's hardcode the values for optimized_theta that we were supposed to get from the minimization function optimized_theta = np.array([-25.16133593, 0.20623171, 0.20147164]) print(optimized_theta)
line_tempt = line_list[i].split(",") x_data[i, :] = line_tempt[:feature_num] y_data[i, 0] = line_tempt[-1] # ========================= Load end ============================= # ========================= plotData ============================= plotData.plot_data(x_data, y_data) # ========================= compute cost and gradient =========== # prepare x_data and initiate theta n = x_data.shape[1] x_data = np.column_stack((np.ones((m, 1)), x_data)) initial_theta = np.zeros((n + 1, 1)).flatten() # must be a vector # cost = costFunction.cost_function(initial_theta, x_data, y_data) # print("initial_theta cost is {} (approx)".format(cost)) # print("expected cost is 0.693") grad = costFunction.gradient(initial_theta.flatten(), x_data, y_data) print("initial_theta grad is {}".format( grad)) #grad can be gotten from costFunction.gradient print('Expected gradients (approx):\n -0.1000\n -12.0092\n -11.2628\n') # ============= Part 3: Optimizing using fminunc(matlab)/scipy(python) ============= # theta must be a vector result = op.minimize(costFunction.cost_function, initial_theta.flatten(), args=(x_data, y_data), method="TNC", jac=costFunction.gradient) final_theta = result.x plotDecisionBoundary.plot_decesion_boundary(final_theta, x_data, y_data) # ============= optional:gradientDescent ============================================ # try the gradientDescent method to optimize theta # final_theta = gradientDescent.gradient_descent(initial_theta, x_data, y_data)
#Ysk = Y.replace(0, 2) Xnp = np.array(X) Ynp = np.array(Y) iters = 50000 # Doesnt saturate after 200,000 itarations also alpha = 0.00001 # 0.00001 upper limit theta = np.zeros(X.shape[1] + 1) # Initialize with zeroes cost_change = [] print("Initial theta:", theta) skmod = LogisticRegression().fit(X, Y) # Learning for i in range(iters): gradient_vector = cf.gradient(theta, Xnp, Ynp) theta = theta - alpha * gradient_vector cost_change.append(cf.cost(theta, Xnp, Ynp)) print("Cost after training:", cf.cost(theta, Xnp, Ynp)) print("Thetas after training:", theta) x_test = np.array([1, 45, 85]) print("Final prediction:", sg.sigmoid(x_test.T.dot(theta))) input = np.array([[45, 85]]) input.reshape(1, -1) print("Final Prediciton SK:", skmod.predict(input)) # Assigning colors to students based on acceptance status colors = [] for i in range(len(Y)): if (Y[i] == 1):
from sigmoid import * data = pd.read_csv('ex2data1.txt', header=None, names=['Exam 1 score', 'Exam 2 score', 'Admission']) # ===================== Part 1: Plotting ===================== plot_data(data) # ===================== Part 2: Compute Cost and Gradient ===================== data.insert(0, 'ones', 1) X = data.values[:,:-1] y = data.values[:,-1] theta = np.zeros(X.shape[1]) cost = cf.cost_function(theta, X, y) grad = cf.gradient(theta, X, y) # ===================== Part 3: Optimizing parameters theta using advanced algorithm ===================== import scipy.optimize as opt result = opt.fmin_tnc(func=cf.cost_function, x0=theta, fprime=cf.gradient, args=(X, y)) theta = result[0] # ===================== Part 4: Predict and Accuracies ===================== predict = sigmoid(np.dot(np.array([1, 45, 85]), theta)) def predict(theta, X): probability = sigmoid(X@theta) return [1 if x >= 0.5 else 0 for x in probability] # return a list