def main(): if os.path.exists('iter_vs_M.pkl'): with open('iter_vs_M.pkl', 'rb') as f: MM = pickle.load(f) iter_gd = pickle.load(f) iter_sgd = pickle.load(f) else: MM = range(11) X, Y = getData(False) iter_gd = [] iter_sgd = [] for M in MM: _, n = gd.fit_polynomial(X, Y, M) iter_gd.append(n) _, n = sgd.fit_polynomial(X, Y, M) iter_sgd.append(n) with open('iter_vs_M.pkl', 'wb') as f: pickle.dump(MM, f) pickle.dump(iter_gd, f) pickle.dump(iter_sgd, f) plt.figure(1, (4.5, 3.5)) plt.plot(MM, iter_gd, 'o', label='BGD', mfc='none') plt.plot(MM, iter_sgd, 'o', label='SGD', mfc='none') plt.xlabel('M') plt.ylabel('Number of iterations') plt.tight_layout() plt.legend(loc='best') plt.savefig('iter_vs_M.png')
def main(): X, Y = getData(False) ndata = len(X) for M in (1, 2, 3, 4, 5, 6, 7, 8): weights = fit_cosines(X, Y, M, 'regress_cos_m_%i.png' % M) print 'M=%i & ' % M, 'w = ', [round(w, 3) for w in weights[:, 0]], '\\\\'
def main(): # for i in range(int(sys.argv[1])): # verify_gradient() poly_fit_degree = int(sys.argv[1]) X_raw, Y_raw = lfd.getData(ifPlotData=False) assert len(X_raw) == len(Y_raw) num_datapoints = len(X_raw) X = np.array(X_raw).reshape(num_datapoints) Y = np.array(Y_raw).reshape(num_datapoints) bgd_weight_vector = gradient_descent( np.zeros(poly_fit_degree + 1), # initial guess lambda w: poly_basis_square_sum_error(w, X, Y), # obj function lambda w: poly_basis_square_sum_error_grad(w, X, Y), 0.05, # step size 1e-5, # convergence threshold 0.05, # gradient approximation parameter (for central diff) False, # no convergence by grad norm stochastic=False) sgd_weight_vector = gradient_descent( np.zeros(poly_fit_degree + 1), # initial guess lambda w, i: poly_basis_square_sum_error(w, np.array([X[ i]]), np.array([Y[i]])), # obj function lambda w, i: poly_basis_square_sum_error_grad(w, np.array([X[i]]), np.array([Y[i]])), 0.01, # step size 1e-5, # convergence threshold 0.05, # gradient approximation parameter (for central diff) False, # no convergence by grad norm stochastic=True)
def p2(): X, Y = getData(ifPlotData=False) Y = np.expand_dims(Y, 1) #X = np.array([.1, .2]) #Y = np.array([1, 2]) M = int(sys.argv[1]) plot = False theta = mle_vector(X, Y, M) phi_X = phi_transform(X, M) params = {'p1': phi_X, 'p2': Y} #sse_val = sse(theta, params) print(finite_difference(sse, theta, params, .0000001)) print(sse_deriv(theta, params)) pdb.set_trace() #theta = np.random.random(11) params = {'p1': phi_X, 'p2': Y} sgd_ans = stochastic_gradient_descent(sse, sse_deriv, .00002, .001, params) pdb.set_trace() if plot == True: plt.plot(X, Y, 'o') plt.xlabel('x') plt.ylabel('y') p = np.poly1d(np.flip(theta, 0)) x = np.arange(0, 1.0, .01) y = p(x) plt.plot(x, y) plt.show()
def training_plot(): X, Y = loadFittingDataP2.getData(ifPlotData='False') plt.scatter(X, Y, facecolors='none', edgecolors='b') x = np.arange(0, 1, 0.001) y = np.cos(x * np.pi) + 1.5 * np.cos(2 * np.pi * x) plt.plot(x, y) plt.xlabel('x') plt.ylabel('y')
def main(): X, Y = getData(False) ndata = len(X) for M in (0,1,2,3,4,6,8,10): #print 'M=%i' % M weights, _ = fit_polynomial(X,Y,M,'sgd_plots/regress_m_%i.png' % M) #print weights print 'SSE = {}'.format(gd.least_squares_objective(weights, gd.polynomial_design_matrix(X, M), Y.reshape((ndata,1)))) print 'SSE derivative = {}'.format(gd.least_squares_gradient(weights, gd.polynomial_design_matrix(X, M), Y.reshape((ndata,1)))) print 'M=%i & ' % M, 'w = ', [round(w,3) for w in weights[:,0]], '\\\\'
def plot_weights(M, out_png): X, Y = getData(False) w_gd, _ = gd.fit_polynomial(X, Y, M) w_sgd, _ = sgd.fit_polynomial(X, Y, M) w = fit_poly.fit_polynomial(X, Y, M) w = w.reshape(M + 1) w_gd = w_gd.reshape(M + 1) w_sgd = w_sgd.reshape(M + 1) plt.figure(1, (3, 3)) plt.clf() width = .3 xx = np.arange(M + 1) plt.bar(xx, w, width, label='analytic') plt.bar(xx + width, w_gd, width, label='BGD') plt.bar(xx + 2 * width, w_sgd, width, label='SGD') plt.title('M={}'.format(M)) plt.legend(loc='best') plt.xlabel('i') plt.ylabel('$w_i$') plt.tight_layout() plt.savefig(out_png)
def plot_cos(X, y, n): theta = max_likelihood_weights_cos(X, y, n) x_n = np.linspace(0, 1, 100) y_n = [ np.dot(theta, [np.cos(math.pi * x_n[i] * j) for j in range(1, n + 1)]) for i in range(100) ] f_n = [ math.cos(3.1415 * i / 100) + math.cos(2 * 3.1415 * i / 100) for i in range(100) ] plt.plot(x_n, y_n, X, y, 'o', x_n, f_n) plt.show() X, y = fitting.getData() X = np.array(X) print X print y ''' print max_likelihood_weights_cos(X,y,1) print max_likelihood_weights_cos(X,y,2) print max_likelihood_weights_cos(X,y,3) print max_likelihood_weights_cos(X,y,4) print max_likelihood_weights_cos(X,y,5) print max_likelihood_weights_cos(X,y,6) print max_likelihood_weights_cos(X,y,7) print max_likelihood_weights_cos(X,y,8) theta=[6,1,9,4]
def SSE_deriv(X, Y, theta): phi_new = poly_phi(X, theta.shape[0]) new = Y - phi_new.dot(theta) SSE = np.sum(np.power(new, 2)) grad = theta_gradient(phi_new, Y, theta) norm_grad = np.linalg.norm(grad) return SSE, grad, norm_grad def theta_gradient(X, Y, theta): return 2 * X.transpose().dot(X.dot(theta) - Y) if __name__ == "__main__": # Load Parameters X_raw, Y_raw = loadFittingDataP2.getData(ifPlotData='False') X = np.transpose(np.matrix(X_raw)) Y = np.transpose(np.matrix(Y_raw)) M = [0, 1, 3, 10] step = 0.01 for i in range(0, len(M)): theta = poly_theta(X, Y, i, function='cosine') plot_ML(X, Y, M[i]) plt.show() # print theta.shape #print SSE_deriv(X, Y, theta) theta = poly_theta(X, Y, 10) SSE_deriv(X, Y, theta)
# # stoch_w, stoch_trials = stochasticGradientDescentWithPlot(sse, np.zeros(10), 100, stepSize) # actual_w = np.linalg.inv(X.T.dot(X)).dot(X.T).dot(y) # print sse.value(actual_w) # # print sse.value(actual_w) # # print "diff is " + str(np.linalg.norm(w - actual_w) / np.linalg.norm(actual_w)) # print "diff is " + str(np.linalg.norm(stoch_w - actual_w) / np.linalg.norm(actual_w)) # # print stoch_w # # print w # # print sse.value(w) # # print trials # print stoch_w # print sse.value(stoch_w) # print stoch_trials # SECOND QUESTION X, Y = loadFittingDataP2.getData(ifPlotData=False) M = 10 w, apply_X, reg_func = poly_regression(X, Y, M) sse = SumSquaredErrors(apply_X, Y) # numericalGradient(sse, np.array([-100000,0,0,0,0,0]), 1) # numericalGradient(sse, np.array([10, 100, 1000]), 1) # numericalGradient(sse, np.array([-500, -200, 100]), 1) def power(i): return lambda a: a**i initial = np.zeros(M + 1) numerical_w, trials = gradientDescent(sse, initial, .044, 4e-4, False) print "finished"
if deriv_func: deriv_squared_error = sum(2 * errors * np.apply_along_axis(deriv_func, 0, X)) return squared_error, deriv_squared_error else: return squared_error def poly_mle(X, Y, M): def power(i): return lambda a: a**i def deriv(i): if i == 0: return lambda a: 0 return lambda a: i * a**(i - 1) basis_funcs = [power(i) for i in range(M)] deriv_funcs = [deriv(i) for i in range(M)] w = mle(X, Y, basis_funcs) return w, basis_funcs, deriv_funcs X, Y = getData(ifPlotData=False) M = 5 w, basis_funcs, deriv_funcs = poly_mle(X, Y, M) reg_func = get_reg_func(w, basis_funcs) deriv_func = get_reg_func(w, deriv_funcs) # plot(X, Y, reg_func) print squared_error(X, Y, reg_func, deriv_func)
x_n = np.linspace(0,1,100) y_n = [np.dot(theta,[x_n[i]**j for j in range(n+1)]) for i in range(100)] yr_n = [np.dot(theta_r,[x_n[i]**j for j in range(n+1)]) for i in range(100)] f_n = [math.cos(3.1415*i/100) + math.cos(2*3.1415*i/100) for i in range(100)] plt.plot(x_n, y_n, X, y, 'o', x_n, f_n, x_n, yr_n) plt.show() def plot_cos(X,y,n): theta = max_likelihood_weights_cos(X,y,n) x_n = np.linspace(0,1,100) y_n = [np.dot(theta,[np.cos(math.pi*x_n[i]*j) for j in range(1,n+1)]) for i in range(100)] f_n = [math.cos(3.1415*i/100) + math.cos(2*3.1415*i/100) for i in range(100)] plt.plot(x_n, y_n, X, y, 'o', x_n, f_n) plt.show() X,y = fitting.getData() X = np.array(X) print X print y ''' print max_likelihood_weights_cos(X,y,1) print max_likelihood_weights_cos(X,y,2) print max_likelihood_weights_cos(X,y,3) print max_likelihood_weights_cos(X,y,4) print max_likelihood_weights_cos(X,y,5) print max_likelihood_weights_cos(X,y,6) print max_likelihood_weights_cos(X,y,7) print max_likelihood_weights_cos(X,y,8) theta=[6,1,9,4]
plt.scatter(X, Y, c='g', label="Test Points", alpha=0.8) plt.title("Ridge Regression for $M = " + str(M) + "$") plt.xlabel('$x$') plt.ylabel('$y$') plt.xlim((-0.01, 1.01)) scalarmappaple = cm.ScalarMappable(norm=normalize, cmap=colormap) scalarmappaple.set_array(M) cb = plt.colorbar(scalarmappaple, ticks=[0, 1, 1.5, 2]) plt.legend() plt.tight_layout() plt.show() if __name__ == '__main__': data = getData(False) # X = data[0] # Y = data[1] # M = [3] lambda_list = [0.1, 0.3, 0.5, 0.7, 0.9, 0.8, 0.6] # # normalize = mcolors.Normalize(vmin=-2, vmax=2) # # colormap =cm.OrRd # # #plot_one(4, X, Y, true_values(X), lambda_list) # # plot_one(10, X, Y, true_values(X), lambda_list) # train_X, train_Y = regressAData() test_X, test_Y = regressBData() val_X, val_Y = validateData() print min_error(M, lambda_list, test_X, test_Y, val_X, val_Y)
t += 1 return previous_values if __name__ == '__main__': #new_x: [ 1.67582053 1.78700602 1.84132746] #new_x: [ 0.01558979 0.54846317 0.91947502] if len(sys.argv) > 1: M = int(sys.argv[1]) else: M = 2 x, y = getData(ifPlotData=False) real_fn = lambda x: math.cos(math.pi * x) + 1.5 * math.cos(2 * math.pi * x) basis0 = lambda x: x**0 basis1 = lambda x: x**1 basis2 = lambda x: x**2 basis3 = lambda x: x**3 basis4 = lambda x: x**4 basis5 = lambda x: x**5 list_of_basis_functions = [basis0, basis1, basis2, basis3, basis4, basis5] ## Maximum likelihood calculations w_mle = calculate_mle_weight(x, y, calculate_polynomial_phi, M) # regression_fn = get_polynomial_regression_fn(w_mle) # fns_to_plot = {
# -*- coding: utf-8 -*- """ Created on Mon Sep 26 18:54:39 2016 @author: loredp """ import functions as f from loadFittingDataP2 import getData import regressData as rg import numpy as np data = getData(ifPlotData=True) ## 3.2 A = rg.regressAData() B = rg.regressBData() v = rg.validateData() X_train = A[0] Y_train = A[1] X_test = B[0] Y_test = B[1] X_val = v[0] Y_val = v[1] M_list = [1,2,5,10] lambda_list = [1e-5,1e-2,0.1,1] mod_A = f.model_eval(X_train,Y_train,M_list,lambda_list) mod_B = f.model_select(X_val,Y_val,M_list,lambda_list, mod_A)
Y_plotting = X_plotting.dot(theta) w, = plt.plot(X_basis, Y_plotting, label="lambda=" + str(l)) plt.legend(handles=[x, y, z, w]) plt.plot(X_o, Y, 'co') plt.xlabel('x') plt.ylabel('y') plt.show() return theta def vandermonde(X, M): #construct a vandermonde matrix with width M. input vars x X_new = np.ones((X.shape[0], M + 1)) for i in xrange(1, M + 1): X_new[:, [i]] = np.matrix(np.power(X, i)) return X_new if __name__ == "__main__": X, Y = loadFittingDataP2.getData(False) X = np.transpose(np.matrix(X)) Y = np.transpose(np.matrix(Y)) L = 100 M = 10 ridge(X, Y, L, M)
def main(): X, Y = getData(False) #Making plots for 3.1 # for M in (0,1,3,10): # for lamb in [10**ii for ii in range(-1,2)]: # weights = fit_polynomial_ridge(X,Y,M,lamb,'ridge_m_%i_lambda_%s.png' % (M,lamb)) #3.2 Xa, Ya = regressAData() Xa = Xa.reshape((13)) Ya = Ya.reshape((13)) Xb, Yb = regressBData() Xb = Xb.reshape((10)) Yb = Yb.reshape((10)) Xval, Yval = validateData() Xval = Xval.reshape((22)) Yval = Yval.reshape((22)) m_lamba = [] train_a = [] test_a_on_b = [] test_a_val = [] train_b = [] test_b_on_a = [] test_b_val = [] data_matrix = [] data_matrix.append([ 'M, Lambda', 'Train A', "Train A/Test B", "Train A/Validate", "Train B", "Train B/Test A", "Train B/Validate" ]) for M in (3, 5, 50): for lamb in [10**ii for ii in range(1, 3)]: m_lamba.append([M, lamb]) weights_A = fit_polynomial_ridge(X=Xa, Y=Ya, M=M, lamb=lamb, return_weights=True) train_a.append(round(R_squared(weights_A, Xa, Ya, M), 2)) test_a_on_b.append(round(R_squared(weights_A, Xb, Yb, M), 2)) test_a_val.append(round(R_squared(weights_A, Xval, Yval, M), 2)) weights_B = fit_polynomial_ridge(X=Xb, Y=Yb, M=M, lamb=lamb, return_weights=True) train_b.append(round(R_squared(weights_B, Xb, Yb, M), 2)) test_b_on_a.append(round(R_squared(weights_B, Xa, Ya, M), 2)) test_b_val.append(round(R_squared(weights_B, Xval, Yval, M), 2)) data_matrix.append([ str([M, lamb]), round(R_squared(weights_A, Xa, Ya, M), 2), round(R_squared(weights_A, Xb, Yb, M), 2), round(R_squared(weights_A, Xval, Yval, M), 2), round(R_squared(weights_B, Xb, Yb, M), 2), round(R_squared(weights_B, Xa, Ya, M), 2), round(R_squared(weights_B, Xval, Yval, M), 2) ]) print data_matrix table = ff.create_table(data_matrix) py.iplot(table, filename='ridge_validation.png')