def main():
if os.path.exists('iter_vs_M.pkl'):
with open('iter_vs_M.pkl', 'rb') as f:
MM = pickle.load(f)
iter_gd = pickle.load(f)
iter_sgd = pickle.load(f)
else:
MM = range(11)
X, Y = getData(False)
iter_gd = []
iter_sgd = []
for M in MM:
_, n = gd.fit_polynomial(X, Y, M)
iter_gd.append(n)
_, n = sgd.fit_polynomial(X, Y, M)
iter_sgd.append(n)
with open('iter_vs_M.pkl', 'wb') as f:
pickle.dump(MM, f)
pickle.dump(iter_gd, f)
pickle.dump(iter_sgd, f)
plt.figure(1, (4.5, 3.5))
plt.plot(MM, iter_gd, 'o', label='BGD', mfc='none')
plt.plot(MM, iter_sgd, 'o', label='SGD', mfc='none')
plt.xlabel('M')
plt.ylabel('Number of iterations')
plt.tight_layout()
plt.legend(loc='best')
plt.savefig('iter_vs_M.png')
def main():
X, Y = getData(False)
ndata = len(X)
for M in (1, 2, 3, 4, 5, 6, 7, 8):
weights = fit_cosines(X, Y, M, 'regress_cos_m_%i.png' % M)
print 'M=%i & ' % M, 'w = ', [round(w, 3)
for w in weights[:, 0]], '\\\\'
def main():
# for i in range(int(sys.argv[1])):
# verify_gradient()
poly_fit_degree = int(sys.argv[1])
X_raw, Y_raw = lfd.getData(ifPlotData=False)
assert len(X_raw) == len(Y_raw)
num_datapoints = len(X_raw)
X = np.array(X_raw).reshape(num_datapoints)
Y = np.array(Y_raw).reshape(num_datapoints)
bgd_weight_vector = gradient_descent(
np.zeros(poly_fit_degree + 1), # initial guess
lambda w: poly_basis_square_sum_error(w, X, Y), # obj function
lambda w: poly_basis_square_sum_error_grad(w, X, Y),
0.05, # step size
1e-5, # convergence threshold
0.05, # gradient approximation parameter (for central diff)
False, # no convergence by grad norm
stochastic=False)
sgd_weight_vector = gradient_descent(
np.zeros(poly_fit_degree + 1), # initial guess
lambda w, i: poly_basis_square_sum_error(w, np.array([X[
i]]), np.array([Y[i]])), # obj function
lambda w, i: poly_basis_square_sum_error_grad(w, np.array([X[i]]),
np.array([Y[i]])),
0.01, # step size
1e-5, # convergence threshold
0.05, # gradient approximation parameter (for central diff)
False, # no convergence by grad norm
stochastic=True)
def p2():
X, Y = getData(ifPlotData=False)
Y = np.expand_dims(Y, 1)
#X = np.array([.1, .2])
#Y = np.array([1, 2])
M = int(sys.argv[1])
plot = False
theta = mle_vector(X, Y, M)
phi_X = phi_transform(X, M)
params = {'p1': phi_X, 'p2': Y}
#sse_val = sse(theta, params)
print(finite_difference(sse, theta, params, .0000001))
print(sse_deriv(theta, params))
pdb.set_trace()
#theta = np.random.random(11)
params = {'p1': phi_X, 'p2': Y}
sgd_ans = stochastic_gradient_descent(sse, sse_deriv, .00002, .001, params)
pdb.set_trace()
if plot == True:
plt.plot(X, Y, 'o')
plt.xlabel('x')
plt.ylabel('y')
p = np.poly1d(np.flip(theta, 0))
x = np.arange(0, 1.0, .01)
y = p(x)
plt.plot(x, y)
plt.show()
def training_plot():
X, Y = loadFittingDataP2.getData(ifPlotData='False')
plt.scatter(X, Y, facecolors='none', edgecolors='b')
x = np.arange(0, 1, 0.001)
y = np.cos(x * np.pi) + 1.5 * np.cos(2 * np.pi * x)
plt.plot(x, y)
plt.xlabel('x')
plt.ylabel('y')
def main():
X, Y = getData(False)
ndata = len(X)
for M in (0,1,2,3,4,6,8,10):
#print 'M=%i' % M
weights, _ = fit_polynomial(X,Y,M,'sgd_plots/regress_m_%i.png' % M)
#print weights
print 'SSE = {}'.format(gd.least_squares_objective(weights, gd.polynomial_design_matrix(X, M), Y.reshape((ndata,1))))
print 'SSE derivative = {}'.format(gd.least_squares_gradient(weights, gd.polynomial_design_matrix(X, M), Y.reshape((ndata,1))))
print 'M=%i & ' % M, 'w = ', [round(w,3) for w in weights[:,0]], '\\\\'
def plot_weights(M, out_png):
X, Y = getData(False)
w_gd, _ = gd.fit_polynomial(X, Y, M)
w_sgd, _ = sgd.fit_polynomial(X, Y, M)
w = fit_poly.fit_polynomial(X, Y, M)
w = w.reshape(M + 1)
w_gd = w_gd.reshape(M + 1)
w_sgd = w_sgd.reshape(M + 1)
plt.figure(1, (3, 3))
plt.clf()
width = .3
xx = np.arange(M + 1)
plt.bar(xx, w, width, label='analytic')
plt.bar(xx + width, w_gd, width, label='BGD')
plt.bar(xx + 2 * width, w_sgd, width, label='SGD')
plt.title('M={}'.format(M))
plt.legend(loc='best')
plt.xlabel('i')
plt.ylabel('$w_i$')
plt.tight_layout()
plt.savefig(out_png)
def plot_cos(X, y, n):
theta = max_likelihood_weights_cos(X, y, n)
x_n = np.linspace(0, 1, 100)
y_n = [
np.dot(theta, [np.cos(math.pi * x_n[i] * j) for j in range(1, n + 1)])
for i in range(100)
]
f_n = [
math.cos(3.1415 * i / 100) + math.cos(2 * 3.1415 * i / 100)
for i in range(100)
]
plt.plot(x_n, y_n, X, y, 'o', x_n, f_n)
plt.show()
X, y = fitting.getData()
X = np.array(X)
print X
print y
'''
print max_likelihood_weights_cos(X,y,1)
print max_likelihood_weights_cos(X,y,2)
print max_likelihood_weights_cos(X,y,3)
print max_likelihood_weights_cos(X,y,4)
print max_likelihood_weights_cos(X,y,5)
print max_likelihood_weights_cos(X,y,6)
print max_likelihood_weights_cos(X,y,7)
print max_likelihood_weights_cos(X,y,8)
theta=[6,1,9,4]
def SSE_deriv(X, Y, theta):
phi_new = poly_phi(X, theta.shape[0])
new = Y - phi_new.dot(theta)
SSE = np.sum(np.power(new, 2))
grad = theta_gradient(phi_new, Y, theta)
norm_grad = np.linalg.norm(grad)
return SSE, grad, norm_grad
def theta_gradient(X, Y, theta):
return 2 * X.transpose().dot(X.dot(theta) - Y)
if __name__ == "__main__":
# Load Parameters
X_raw, Y_raw = loadFittingDataP2.getData(ifPlotData='False')
X = np.transpose(np.matrix(X_raw))
Y = np.transpose(np.matrix(Y_raw))
M = [0, 1, 3, 10]
step = 0.01
for i in range(0, len(M)):
theta = poly_theta(X, Y, i, function='cosine')
plot_ML(X, Y, M[i])
plt.show()
# print theta.shape
#print SSE_deriv(X, Y, theta)
theta = poly_theta(X, Y, 10)
SSE_deriv(X, Y, theta)
# # stoch_w, stoch_trials = stochasticGradientDescentWithPlot(sse, np.zeros(10), 100, stepSize)
# actual_w = np.linalg.inv(X.T.dot(X)).dot(X.T).dot(y)
# print sse.value(actual_w)
# # print sse.value(actual_w)
# # print "diff is " + str(np.linalg.norm(w - actual_w) / np.linalg.norm(actual_w))
# print "diff is " + str(np.linalg.norm(stoch_w - actual_w) / np.linalg.norm(actual_w))
# # print stoch_w
# # print w
# # print sse.value(w)
# # print trials
# print stoch_w
# print sse.value(stoch_w)
# print stoch_trials
# SECOND QUESTION
X, Y = loadFittingDataP2.getData(ifPlotData=False)
M = 10
w, apply_X, reg_func = poly_regression(X, Y, M)
sse = SumSquaredErrors(apply_X, Y)
# numericalGradient(sse, np.array([-100000,0,0,0,0,0]), 1)
# numericalGradient(sse, np.array([10, 100, 1000]), 1)
# numericalGradient(sse, np.array([-500, -200, 100]), 1)
def power(i):
return lambda a: a**i
initial = np.zeros(M + 1)
numerical_w, trials = gradientDescent(sse, initial, .044, 4e-4, False)
print "finished"
if deriv_func:
deriv_squared_error = sum(2 * errors *
np.apply_along_axis(deriv_func, 0, X))
return squared_error, deriv_squared_error
else:
return squared_error
def poly_mle(X, Y, M):
def power(i):
return lambda a: a**i
def deriv(i):
if i == 0:
return lambda a: 0
return lambda a: i * a**(i - 1)
basis_funcs = [power(i) for i in range(M)]
deriv_funcs = [deriv(i) for i in range(M)]
w = mle(X, Y, basis_funcs)
return w, basis_funcs, deriv_funcs
X, Y = getData(ifPlotData=False)
M = 5
w, basis_funcs, deriv_funcs = poly_mle(X, Y, M)
reg_func = get_reg_func(w, basis_funcs)
deriv_func = get_reg_func(w, deriv_funcs)
# plot(X, Y, reg_func)
print squared_error(X, Y, reg_func, deriv_func)
x_n = np.linspace(0,1,100) y_n = [np.dot(theta,[x_n[i]**j for j in range(n+1)]) for i in range(100)] yr_n = [np.dot(theta_r,[x_n[i]**j for j in range(n+1)]) for i in range(100)] f_n = [math.cos(3.1415*i/100) + math.cos(2*3.1415*i/100) for i in range(100)] plt.plot(x_n, y_n, X, y, 'o', x_n, f_n, x_n, yr_n) plt.show() def plot_cos(X,y,n): theta = max_likelihood_weights_cos(X,y,n) x_n = np.linspace(0,1,100) y_n = [np.dot(theta,[np.cos(math.pi*x_n[i]*j) for j in range(1,n+1)]) for i in range(100)] f_n = [math.cos(3.1415*i/100) + math.cos(2*3.1415*i/100) for i in range(100)] plt.plot(x_n, y_n, X, y, 'o', x_n, f_n) plt.show() X,y = fitting.getData() X = np.array(X) print X print y ''' print max_likelihood_weights_cos(X,y,1) print max_likelihood_weights_cos(X,y,2) print max_likelihood_weights_cos(X,y,3) print max_likelihood_weights_cos(X,y,4) print max_likelihood_weights_cos(X,y,5) print max_likelihood_weights_cos(X,y,6) print max_likelihood_weights_cos(X,y,7) print max_likelihood_weights_cos(X,y,8) theta=[6,1,9,4]
plt.scatter(X, Y, c='g', label="Test Points", alpha=0.8)
plt.title("Ridge Regression for $M = " + str(M) + "$")
plt.xlabel('$x$')
plt.ylabel('$y$')
plt.xlim((-0.01, 1.01))
scalarmappaple = cm.ScalarMappable(norm=normalize, cmap=colormap)
scalarmappaple.set_array(M)
cb = plt.colorbar(scalarmappaple, ticks=[0, 1, 1.5, 2])
plt.legend()
plt.tight_layout()
plt.show()
if __name__ == '__main__':
data = getData(False)
# X = data[0]
# Y = data[1]
#
M = [3]
lambda_list = [0.1, 0.3, 0.5, 0.7, 0.9, 0.8, 0.6]
# # normalize = mcolors.Normalize(vmin=-2, vmax=2)
# # colormap =cm.OrRd
# # #plot_one(4, X, Y, true_values(X), lambda_list)
# # plot_one(10, X, Y, true_values(X), lambda_list)
#
train_X, train_Y = regressAData()
test_X, test_Y = regressBData()
val_X, val_Y = validateData()
print min_error(M, lambda_list, test_X, test_Y, val_X, val_Y)
t += 1
return previous_values
if __name__ == '__main__':
#new_x: [ 1.67582053 1.78700602 1.84132746]
#new_x: [ 0.01558979 0.54846317 0.91947502]
if len(sys.argv) > 1:
M = int(sys.argv[1])
else:
M = 2
x, y = getData(ifPlotData=False)
real_fn = lambda x: math.cos(math.pi * x) + 1.5 * math.cos(2 * math.pi * x)
basis0 = lambda x: x**0
basis1 = lambda x: x**1
basis2 = lambda x: x**2
basis3 = lambda x: x**3
basis4 = lambda x: x**4
basis5 = lambda x: x**5
list_of_basis_functions = [basis0, basis1, basis2, basis3, basis4, basis5]
## Maximum likelihood calculations
w_mle = calculate_mle_weight(x, y, calculate_polynomial_phi, M)
# regression_fn = get_polynomial_regression_fn(w_mle)
# fns_to_plot = {
# -*- coding: utf-8 -*- """ Created on Mon Sep 26 18:54:39 2016 @author: loredp """ import functions as f from loadFittingDataP2 import getData import regressData as rg import numpy as np data = getData(ifPlotData=True) ## 3.2 A = rg.regressAData() B = rg.regressBData() v = rg.validateData() X_train = A[0] Y_train = A[1] X_test = B[0] Y_test = B[1] X_val = v[0] Y_val = v[1] M_list = [1,2,5,10] lambda_list = [1e-5,1e-2,0.1,1] mod_A = f.model_eval(X_train,Y_train,M_list,lambda_list) mod_B = f.model_select(X_val,Y_val,M_list,lambda_list, mod_A)
Y_plotting = X_plotting.dot(theta)
w, = plt.plot(X_basis, Y_plotting, label="lambda=" + str(l))
plt.legend(handles=[x, y, z, w])
plt.plot(X_o, Y, 'co')
plt.xlabel('x')
plt.ylabel('y')
plt.show()
return theta
def vandermonde(X, M):
#construct a vandermonde matrix with width M. input vars x
X_new = np.ones((X.shape[0], M + 1))
for i in xrange(1, M + 1):
X_new[:, [i]] = np.matrix(np.power(X, i))
return X_new
if __name__ == "__main__":
X, Y = loadFittingDataP2.getData(False)
X = np.transpose(np.matrix(X))
Y = np.transpose(np.matrix(Y))
L = 100
M = 10
ridge(X, Y, L, M)
def main():
X, Y = getData(False)
#Making plots for 3.1
# for M in (0,1,3,10):
# for lamb in [10**ii for ii in range(-1,2)]:
# weights = fit_polynomial_ridge(X,Y,M,lamb,'ridge_m_%i_lambda_%s.png' % (M,lamb))
#3.2
Xa, Ya = regressAData()
Xa = Xa.reshape((13))
Ya = Ya.reshape((13))
Xb, Yb = regressBData()
Xb = Xb.reshape((10))
Yb = Yb.reshape((10))
Xval, Yval = validateData()
Xval = Xval.reshape((22))
Yval = Yval.reshape((22))
m_lamba = []
train_a = []
test_a_on_b = []
test_a_val = []
train_b = []
test_b_on_a = []
test_b_val = []
data_matrix = []
data_matrix.append([
'M, Lambda', 'Train A', "Train A/Test B", "Train A/Validate",
"Train B", "Train B/Test A", "Train B/Validate"
])
for M in (3, 5, 50):
for lamb in [10**ii for ii in range(1, 3)]:
m_lamba.append([M, lamb])
weights_A = fit_polynomial_ridge(X=Xa,
Y=Ya,
M=M,
lamb=lamb,
return_weights=True)
train_a.append(round(R_squared(weights_A, Xa, Ya, M), 2))
test_a_on_b.append(round(R_squared(weights_A, Xb, Yb, M), 2))
test_a_val.append(round(R_squared(weights_A, Xval, Yval, M), 2))
weights_B = fit_polynomial_ridge(X=Xb,
Y=Yb,
M=M,
lamb=lamb,
return_weights=True)
train_b.append(round(R_squared(weights_B, Xb, Yb, M), 2))
test_b_on_a.append(round(R_squared(weights_B, Xa, Ya, M), 2))
test_b_val.append(round(R_squared(weights_B, Xval, Yval, M), 2))
data_matrix.append([
str([M, lamb]),
round(R_squared(weights_A, Xa, Ya, M), 2),
round(R_squared(weights_A, Xb, Yb, M), 2),
round(R_squared(weights_A, Xval, Yval, M), 2),
round(R_squared(weights_B, Xb, Yb, M), 2),
round(R_squared(weights_B, Xa, Ya, M), 2),
round(R_squared(weights_B, Xval, Yval, M), 2)
])
print data_matrix
table = ff.create_table(data_matrix)
py.iplot(table, filename='ridge_validation.png')
