def gd(init, tol, iteration):
result = [init]
guess_eval = -answers.lml(init[0], init[1], Phi, Y)
guess_grad = -answers.grad_lml(init[0], init[1], Phi, Y)
guess_grads = [guess_grad]
guess_evals = [guess_eval]
convsgd = []
lenXgd = []
diffFsd = []
print("iter={}, func val={}".format(0, guess_evals[0]))
for i in range(1, iteration):
guess = result[i - 1] - step_size * guess_grads[i - 1]
result.append(guess)
guess_eval = -answers.lml(guess[0], guess[1], Phi, Y)
guess_grad = -answers.grad_lml(guess[0], guess[1], Phi, Y)
guess_evals.append(guess_eval)
guess_grads.append(guess_grad)
print("iter={}, func val={}".format(i, guess_evals[i]))
convsgd.append(np.linalg.norm(guess_grad))
lenXgd.append(np.linalg.norm(result[-1] - result[-2]))
diffFsd.append(np.abs(guess_evals[-1] - guess_evals[-2]))
if convsgd[-1] <= tol:
print("First-Order Optimality Condition met")
break
#elif lenXgd[-1] <= tol:
#print("Design not changing")
#break
#elif diffFsd[-1] <= tol:
#print("Objective not changing")
#break
elif i + 1 >= iteration:
print("Done iterating")
break
return np.array(result)
def grad(x, y):
gama = 0.00005
v = True
count = 0
mlml = 0
while (v == True):
#for i in range(70):
s1 = x
s2 = y
#print('s1 :',s1)
#print('s2 :',s2)
g1 = an.grad_lml(x, y, phi, Yd)[0]
g2 = an.grad_lml(x, y, phi, Yd)[1]
x = x + (gama) * g1
y = y + (gama) * g2
t1 = (gama) * g1
t2 = (gama) * g2
#print('t1 :',t1)
#print('t2 :',t2)
ax.arrow(s1,
s2,
t1,
t2,
head_width=0.009,
head_length=0.01,
fc='k',
ec='k')
#if(i==49):
#plot2Dpoint(x,y)
mlml1 = an.lml(x, y, phi, Yd)
if (mlml1 - mlml == 0 or count > 10000):
if (mlml1 - mlml < 0.00001):
print(mlml1, x, y)
v = False
mlml = mlml1
count = count + 1
#print("Objective not changing")
#break
elif i + 1 >= iteration:
print("Done iterating")
break
return np.array(result)
tol = 10**-12
result = gd(np.array([0.46, 0.46]), tol, 50000)
print(result[-1])
print(result.shape)
density = 150
x = np.linspace(0.41, 0.465, density)
y = np.linspace(0.44, 0.461, density)
xaxis, yaxis = np.meshgrid(x, y)
zaxis = np.zeros([density, density])
for k in range(density):
for j in range(density):
zaxis[k, j] = answers.lml(xaxis[k, j], yaxis[k, j], Phi, Y)
con = plt.contour(xaxis, yaxis, zaxis, 30)
plt.clabel(con, inline=1, fontsize=8)
plt.plot(result[:, 0], result[:, 1])
plt.xlabel(r'$\alpha$', fontsize=14)
plt.ylabel(r'$\beta$', fontsize=14)
plt.title('1-order polynomial with step size ' + str(step_size))
plt.show()
Xd = np.linspace(0, 0.9, 25)
Yd = np.cos(10 * Xd**2) + 0.1 * np.sin(100 * Xd)
x0 = [1] * 25
x1 = Xd
phi = np.column_stack((x0, x1))
#phi = np.reshape([[1]*25, Xd], [25,2])
delta = 0.005
a = np.arange(0.3, 1, delta) #M
b = np.arange(0.3, 1, delta) #N
m = a.shape[0]
n = b.shape[0]
z = np.empty([n, m])
for a1 in range(m):
for b1 in range(n):
z[b1][a1] = an.lml(a[a1], b[b1], phi, Yd)
#print(z.shape)
A, B = np.meshgrid(a, b)
Z = np.empty((
len(A),
len(B),
))
for a in range(len(A)):
for b in range(len(B)):
Z[a][b] = an.lml(A[a][b], B[a][b], phi, Yd)
fig, ax = plt.subplots()
CS = ax.contour(A, B, Z, 30)
ax.clabel(CS, inline=1, fontsize=10)
ax.set_title('Maximize Log marginal Likelihood - Linear Function Basis')
J = j / 2
phi[i][j] = np.cos(2 * np.pi * J * X[i])
return phi
phi = phi2(25, 9) # order!
delta = 0.005 #0.005[1-4]
a = np.arange(0.01, 0.5, delta) #M
b = np.arange(0.01, 0.5, delta) #N
m = a.shape[0]
n = b.shape[0]
z = np.empty([n, m])
for a1 in range(m):
for b1 in range(n):
z[b1][a1] = an.lml(a[a1], b[b1], phi, Yd)
#print(z.shape)
fig, ax = plt.subplots()
CS = ax.contour(a, b, z, 50)
ax.clabel(CS, inline=1, fontsize=10)
ax.set_title('Maximize Log marginal Likelihood - Linear Function Basis')
#(starting point-(0.9,0.9) step_size = 0.025)
ax.set_xlabel('alpha >= 0')
ax.set_ylabel('beta >= 0')
def plot2Dpoint(x, y):
ax.hold(True)
plt.scatter(x, y)
def gd(init, tol, iteration, order):
Phi = generate_for_tri(X, order)
result = [init]
guess_eval = -answers.lml(init[0], init[1], Phi, Y)
guess_grad = -answers.grad_lml(init[0], init[1], Phi, Y)
guess_grads = [guess_grad]
guess_evals = [guess_eval]
step = 0.01
while (step * guess_grad[0] >= init[0] or step * guess_grad[1] >= init[1]):
step *= 0.5
tmp = -answers.lml(init[0] - step*guess_grad[0], \
init[1] - step*guess_grad[1], Phi, Y)
while (tmp > guess_eval):
step *= 0.5
tmp = -answers.lml(init[0] - step*guess_grad[0], \
init[1] - step*guess_grad[1], Phi, Y)
step_size = step
convsgd = []
lenXgd = []
diffFsd = []
print("iter={}, func val={}, alpha={}, beta={}".format(
0, guess_eval, init[0], init[1]))
for i in range(1, iteration + 1):
#step_size /= np.sqrt(i)
guess = result[i - 1] - step_size * guess_grads[i - 1]
result.append(guess)
guess_eval = -answers.lml(guess[0], guess[1], Phi, Y)
guess_grad = -answers.grad_lml(guess[0], guess[1], Phi, Y)
guess_evals.append(guess_eval)
guess_grads.append(guess_grad)
if guess_eval >= 0 and i % 1000 == 0:
print("iter={}, func val={}, alpha={}, beta={}".format(
i, guess_eval, guess[0], guess[1]))
convsgd.append(np.linalg.norm(guess_grad))
lenXgd.append(np.linalg.norm(result[-1] - result[-2]))
diffFsd.append(np.abs(guess_evals[-1] - guess_evals[-2]))
if convsgd[-1] <= tol:
print("First-Order Optimality Condition met")
break
elif lenXgd[-1] <= tol:
print("Design not changing")
break
#elif diffFsd[-1] <= 0:
#print("Objective not changing")
#break
elif i + 1 >= iteration:
print("Done iterating")
break
step = 0.1
while (step * guess_grad[0] > guess[0]
or step * guess_grad[1] > guess[1]):
step *= 0.5
m = np.linalg.norm(guess_grad)
tmp = -answers.lml(guess[0] - step*guess_grad[0], \
guess[1] - step*guess_grad[1], Phi, Y)
while (tmp > guess_eval - step * 0.01 * m):
step *= 0.5
tmp = -answers.lml(guess[0] - step*guess_grad[0], \
guess[1] - step*guess_grad[1], Phi, Y)
step_size = step
print("iter={}, func val={}, alpha={}, beta={}".format(
i, guess_eval, guess[0], guess[1]))
return [np.array(result), guess_eval]
def plot(maximums, orders):
plt.plot(orders, maximums)
plt.xlabel('$order$')
plt.ylabel('$max-lml$')
plt.show()
if __name__ == '__main__':
maximas = []
maximums = []
maximum_order = 11
orders = range(maximum_order + 1)
for order in orders:
Phi = trig_func(order, X) # init feature matrix
path = gradient_descent(np.array([.35, .35]), 1e-5, Phi)
maxima = (path[0, -1], path[1, -1])
maximum = lml(maxima[0], maxima[1], Phi, Y)
maximas.append(maxima)
maximums.append(maximum)
assert len(maximas) == maximum_order + 1
assert len(maximums) == maximum_order + 1
# plot for (c), a graph of max-lml vs order
plot(maximums, orders)