def p1():
mu, cov, quad_A, quad_b = getData()
#batch_gradient_descent(neg_gaussian, neg_gaussian_deriv, .01, .000001, mu, cov, guess=np.array([-1, 20]))
#batch_gradient_descent(quad_bowl, quad_bowl_deriv, .01, .000001, quad_A, quad_b, guess=np.array([-1, 20]))
#mu = np.array([10, 10])
#cov = np.array([[10, 0],[0,10]])
x = np.array([10, 20])
params = {'p1': quad_A, 'p2': quad_b}
print finite_difference(quad_bowl, np.array([1, 1]), params, .001)
print quad_bowl_deriv(np.array([1, 1]), params)
X, y = loadFittingDataP1.getData()
#test_X = np.array([[1,2],[4,6],[-1,5]])
#test_y = np.array([3,10,4])
params = {'p1': X, 'p2': y}
#batch_gradient_descent(least_squares, least_squares_deriv, .00001, .0001, X, y)
stochastic_gradient_descent(least_squares, least_squares_deriv, .00002,
.001, params)
print correct_soln(X, y)
from loadParametersP1 import getData
import math
from loadFittingDataP1 import getData as getFittingData
import random
def gradient_descent(f, f_prime, x0, step, treshold, maxiter=500):
x = x0
iter = 0
while np.linalg.norm(f_prime(x)) > treshold & iter < maxiter:
x = x - step * f_prime(x)
iter += 1
return x
parametersP1 = getData()
fittingData = getFittingData()
u = np.matrix.transpose(np.asmatrix(parametersP1[0]))
sigma = np.asmatrix(parametersP1[1])
A = np.asmatrix(parametersP1[2])
b = np.matrix.transpose(np.asmatrix(parametersP1[3]))
def gaussian_function(x):
n = len(u)
return (-1 / (math.sqrt(
((2 * math.pi)**n)) * np.linalg.det(sigma))) * math.exp(
-np.matrix.transpose(x - u) * np.linalg.inv(sigma) * (x - u) / 2)
def gaussian_function_prime(x):
def get_quad_params():
a, b = loadParams.getData()[2:4]
b = np.array(b).reshape(2)
a = np.array(a)
return (a, b)
def get_gaussian_params():
mu, sigma = loadParams.getData()[:2]
mu = np.array(mu).reshape(2)
sigma = np.array(sigma)
return (mu, sigma)
x_n=[10**i for i in range (-5,5)]
l_n=[i for i in range(-5,5)]
y_n=[np.linalg.norm(gradient_descent(func,func_deriv,init_theta,step_size,x)[0]-[[26.67],[26.67]]) for x in x_n]
plt.plot(l_n,y_n)
plt.title('Quadratic Bowl')
plt.xlabel('log(threshold)')
plt.ylabel('norm(diff)')
plt.show()
y_n=[gradient_descent(func,func_deriv,init_theta,step_size,x)[2] for x in x_n]
plt.plot(l_n,y_n)
plt.title('Quadratic Bowl')
plt.xlabel('log(threshold)')
plt.ylabel('iterations')
plt.show()
gaussMean,gaussCov,quadBowlA,quadBowlb = parameters.getData()
neg_gaussian_mean = 0
neg_gaussian_cov = 0
neg_gaussian_coef = 0
neg_gaussian_inv = 0
init_globals_gaussian(gaussMean, gaussCov)
#print "Running gradient descent on negative gaussian:"
#plot_iter(neg_gaussian, neg_gaussian_deriv, [[0],[0]],10**7,10**(-10))
#print gradient_descent(neg_gaussian, neg_gaussian_deriv, [[-1],[-1]],10**7, 10**(-10))[2]
#print "Running gradient approx on negative gaussian:"
quad_A = 0
quad_b = 0
init_globals_quad(quadBowlA, quadBowlb)
'''
def main():
# Load params
gaussMean, gaussCov, quadBowlA, quadBowlb = load_params.getData()
# Negative gaussian
print "True mean:", gaussMean
f = negative_gaussian
multiplier = 1e4
f_params = [gaussMean, gaussCov, multiplier]
learning_rate = 0.01
convergence_limit = 1e-5
max_iters = 500
initial_guess = np.array([8.0, 8.0])
colors = ['b', 'r', 'g', 'y', 'c', 'p', 'b']
marker = []
# Plot to show effect of initial guess.
fig = plt.figure()
initial_guesses = [
np.array([8.0, 18.0]),
np.array([12.0, 16.0]),
np.array([8.0, 8.0])
]
legend = []
for i, initial_guess in enumerate(initial_guesses):
x, fx, dx, num_iters = gradient_descent(f, f_params, learning_rate,
convergence_limit,
initial_guess, max_iters)
plt.plot(range(num_iters + 1), fx, '-o', c=colors[i])
legend.append('Initial Guess Error Norm: ' +
str(round(np.linalg.norm(initial_guess - x[-1]), 2)))
plt.legend(legend)
plt.xlabel("Iteration Number")
plt.ylabel("f(x) [Negative Gaussian]")
plt.show()
# Plot to show effect of step size on convergence
fig = plt.figure()
convergence_limits = [1e-4, 1e-3, 1e-2, 1e-1]
legend = []
for i, convergence_limit in enumerate(convergence_limits):
x, fx, dx, num_iters = gradient_descent(f, f_params, learning_rate,
convergence_limit,
initial_guess, max_iters)
plt.plot(range(num_iters + 1), fx, '-o', c=colors[i])
legend.append('Convergence Limit: ' + str(convergence_limit))
plt.legend(legend)
plt.xlabel("Iteration Number")
plt.ylabel("f(x) [Negative Gaussian]")
plt.show()
fig = plt.figure()
convergence_limits = np.logspace(2, -3, 20)
print convergence_limits
legend = []
solutions = []
for i, convergence_limit in enumerate(convergence_limits):
x, fx, dx, num_iters = gradient_descent(f, f_params, learning_rate,
convergence_limit,
initial_guess, max_iters)
solutions.append(fx[-1])
plt.semilogx(convergence_limits, solutions, 'o')
plt.xlabel("Convergence Limit")
plt.ylabel("f(x) [Negative Gaussian]")
plt.show()
# Quadratic Bowl
f = quadratic_bowl
f_params = [quadBowlA, quadBowlb]
learning_rate = 0.01
convergence_limit = 1e-1
initial_guess = np.array([2.0, -2.0])
max_iters = 20
x, fx, dx, num_iters = gradient_descent(f, f_params, learning_rate,
convergence_limit, initial_guess,
max_iters)
print '--------'
print "f([10,10]):", f(f_params, np.array([10, 10]))
print "x:", x
print "fx:", fx
print "dx:", dx
print "minimum: (%s,%s). Took %s steps." % (x[-1], fx[-1], np.shape(x)[0])
fig = plt.figure()
plt.scatter(range(num_iters + 1), fx)
plt.xlabel("Iteration Number")
plt.ylabel("f(x)")
plt.show()
fig2 = plt.figure()
plt.plot(range(num_iters + 1), np.linalg.norm(dx, axis=1), '-o')
plt.xlabel("Iteration Number")
plt.ylabel("Norm of Gradient [Quadratic Bowl]")
plt.show()
def negGaussianGrad(x):
# return -1*negGaussian(x)*np.linalg.inv(cov)*(x-mu)
return np.linalg.inv(cov) * (x - mu) * -1 * negGaussian(x)
if __name__ == "__main__":
print "running gradientDescent.py"
global quadBowlA
global quadBowlb
global mu
global cov
(mu, cov, quadBowlA, quadBowlb) = loadParametersP1.getData()
mu = np.transpose(np.matrix(mu))
quadBowlb = np.transpose(np.matrix(quadBowlb))
#data = mean, Gaussian covariance, A and b for quadratic bowl in order
guess = np.transpose(np.matrix([[20, 10]])) #specify guess here
step = 0.05 #specify step here
conv = 0.0001 #specify convergence here
h = 1
print "mean", mu
print "gauss cov", cov
print "Quadratic Bowl A", quadBowlA
print "Quadratic Bowl B", quadBowlb
x, y, i = gradientDescent(quadraticBowl, quadraticBowlGrad, guess, 0.1,
plt.title('Quadratic Bowl')
plt.xlabel('log(threshold)')
plt.ylabel('norm(diff)')
plt.show()
y_n = [
gradient_descent(func, func_deriv, init_theta, step_size, x)[2]
for x in x_n
]
plt.plot(l_n, y_n)
plt.title('Quadratic Bowl')
plt.xlabel('log(threshold)')
plt.ylabel('iterations')
plt.show()
gaussMean, gaussCov, quadBowlA, quadBowlb = parameters.getData()
neg_gaussian_mean = 0
neg_gaussian_cov = 0
neg_gaussian_coef = 0
neg_gaussian_inv = 0
init_globals_gaussian(gaussMean, gaussCov)
#print "Running gradient descent on negative gaussian:"
#plot_iter(neg_gaussian, neg_gaussian_deriv, [[0],[0]],10**7,10**(-10))
#print gradient_descent(neg_gaussian, neg_gaussian_deriv, [[-1],[-1]],10**7, 10**(-10))[2]
#print "Running gradient approx on negative gaussian:"
quad_A = 0
quad_b = 0
init_globals_quad(quadBowlA, quadBowlb)
'''
def main():
gaussMean,gaussCov,quadBowlA,quadBowlb = loadParametersP1.getData()
gaussMean = gaussMean.reshape((2,1))
quadBowlb = quadBowlb.reshape((2,1))
# print gaussMean
# print gaussCov
# maxval = 20
# minval = 0
# step = 1
# x1, x2 = np.mgrid[slice(minval, maxval, step), slice(minval, maxval, step)]
# z = np.empty(((maxval-minval)/step,(maxval-minval)/step))
# for i in range(0,len(x2)):
# for j in range(0,len(x2[0])):
# x_ij = np.array([x1[i][j],x2[i][j]])
# z[i][j] = np.log(np.linalg.norm(gaussian_gradient(gaussMean, gaussCov, x_ij)- gradient_descent.central_difference(lambda x: gaussian_objective(gaussMean, gaussCov,x),1, x_ij).reshape((2,1))))
# z_min, z_max = -np.abs(z).max(), np.abs(z).max()
# z = (1 - x1 / 2. + x1 ** 5 + x2 ** 3) * np.exp(-x1 ** 2 - x2 ** 2)
# print np.log(z)
# fig, ax = plt.subplots()
# plt.pcolor(x1, x2, np.log(z), cmap='RdBu')
# plt.title('pcolor')
# # set the limits of the plot to the limits of the data
# plt.axis([x1.min(), x1.max(), x2.min(), x2.max()])
# plt.colorbar()
# plt.savefig("test_heatmap.png")
#####################################################################################
# Effect of start guess
#####################################################################################
# # Quadratic
# start_guess(objective=quadratic_objective, gradient=quadratic_gradient, obj_params=[quadBowlA,quadBowlb],
# points=[[(80/3.)-10.,(80/3.)-10.],[(80/3.)-10.,(80/3.)+10.],[(80/3.),(80/3.)-20.],[(80/3.)-20.,(80/3.)-20.],[30.,30.]],
# cols = ['b^','ko','r-','cv','g>'], h=10**(-2),cc=10**(1),func_type="Quadratic",out_png="quadratic_starting.png")
# # #Notes, blue 16,15 converges faster than black 16,26 even tho it starts off worse, which makes sense due to
# # #co-dependence
# # #otherwise, close things pretty much
# # Gaussian
# start_guess(objective=gaussian_objective, gradient=gaussian_gradient, obj_params=[gaussMean,gaussCov],
# points=[[5.,10.],[5.,15.],[15.,15.],[2.5,10.],[12.5,12.5]],
# cols = ['b^','ko','r-','cv','g>'], h=10**(-2),cc=10**(-2),func_type="Gaussian",out_png="gaussian_starting.png")
# #####################################################################################
# # Effect of step size
# #####################################################################################
# # Quadratic
# step_size_effect(objective=quadratic_objective, gradient=quadratic_gradient, obj_params=[quadBowlA,quadBowlb],
# steps=[0.001,0.005,0.01,0.05,0.1],
# cols = ['b^','ko','r-','cv','g>'], start_guess=[(80/3.)-10,(80/3)-10],
# cc=10**(1),func_type="Quadratic",out_png="quadratic_step_size.png")
# # Gaussian
# step_size_effect(objective=gaussian_objective, gradient=gaussian_gradient, obj_params=[gaussMean,gaussCov],
# steps=[0.001,0.005,0.01,0.05,0.1],
# cols = ['b^','ko','r-','cv','g>'], start_guess=[12,12],
# cc=10**(-2),func_type="Gaussian",out_png="guassian_step_size.png")
# #####################################################################################
# # Effect of Convergence Criteria
# #####################################################################################
# #Not so sure that I actually want to go with these plots, if I have time, convert to num iterations
# # vs distance from the global minimum and then label the points in the plot
# # Quadratic
# covergence_criteria_effect(objective=quadratic_objective, gradient=quadratic_gradient,
# obj_params=[quadBowlA,quadBowlb],ccs=[0.001,0.01,0.1,1],
# cols = ['b^','ko','r-','cv','g>'], start_guess=[(80/3.)-10,(80/3)-10],
# step=10**(-2),func_type="Quadratic",out_png="quadratic_convergence_criteria.png")
# # Gaussian
# covergence_criteria_effect(objective=gaussian_objective, gradient=gaussian_gradient,
# obj_params=[gaussMean,gaussCov],ccs=[0.001,0.01,0.1,1],
# cols = ['b^','ko','r-','cv','g>'], start_guess=[12,12],
# step=10**(-2),func_type="Gaussian",out_png="gaussian_convergence_criteria.png")
# #####################################################################################
# # Evolution of the norm of the gradient
# #####################################################################################
# plot_norm_of_gradient(objective=quadratic_objective, gradient=quadratic_gradient,
# obj_args= [quadBowlA,quadBowlb],start_guess=np.array([(80/3.)-10,(80/3)-10]), step=10**(-2),
# cc=10**1, func_type="Quadratic", out_png="quadratic_gradient_norm.png")
# plot_norm_of_gradient(objective=gaussian_objective, gradient=gaussian_gradient,
# obj_args= [gaussMean,gaussCov],start_guess=np.array([12,12]), step=10**(-2),
# cc=10**(-2), func_type="Gaussian", out_png="gaussian_gradient_norm.png")
#####################################################################################
# 1.2 Central Difference Calculations
#####################################################################################
plot_finite_diff_vs_gradient(objective=quadratic_objective, gradient=quadratic_gradient,
obj_args= [quadBowlA,quadBowlb], finite_steps = [0.1,1,10,1000,100000], cols = ['b-','k-','r-','c-','g-'],
start_guess=np.array([(80/3.)-100,(80/3)-100]), step=10**(-2),
cc=10**1, func_type="Quadratic", out_png="quadratic_finite_difference.png")
#notes: a sweet spot occurs due to the addition and division terms
#but still really low in all cases, on the scale of 10^-10
plot_finite_diff_vs_gradient(objective=gaussian_objective, gradient=gaussian_gradient,
obj_args= [gaussMean, gaussCov], finite_steps = [1,2.5,5,10,100], cols = ['b-','k-','r-','c-','g-'],
start_guess=np.array([12,12]), step=10**(-2),
cc=10**(-2), func_type="Gaussian", out_png="gaussian_finite_difference.png")
ax = fig.add_subplot(1, 1, 1)
plt.plot(X, Y, 'o')
plt.plot(x_cont, y_pred1, 'c')
# plt.plot(x_cont, y_original)
plt.plot(x_cont, y_pred2, 'm')
# plt.plot(x_cont, y_pred3)
plt.title(u'Solution by Gradient Descent (M = ' + str(M) + ')')
plt.xlabel('x')
plt.ylabel('y')
plt.show()
if __name__ == '__main__':
gaussMean, gaussCov, quadBowlA, quadBowlB = loadParametersP1.getData()
gauss = NegativeGaussian(gaussMean, gaussCov)
quad = QuadraticBowl(quadBowlA, quadBowlB)
quadSolution = np.array([80. / 3, 80. / 3])
# NUMBER 1
# initialGuesses = [np.array([9.9, 9.9]), np.array([9,9]), np.array([0,0]), np.array([-90, -90]), np.array([-990, -990])]
# for initial in initialGuesses:
# guess, numTrials = gradientDescent(gauss, initial, 1e6, 1e-9, False)
# print "Gave " + str(error(guess, gaussMean)) + " error with " + str(numTrials) + " trials with initial " + str(initial)
# print ""
# stepSizes = [1e4, 1e5, 1e6, 1e7, 1e8]
# for step in stepSizes:
new_x = numpy.copy(x)
new_value = new_x[i] + delta1
numpy.put(old_x, [i], [old_value])
numpy.put(new_x, [i], [new_value])
current_slope = (f(new_x) - f(old_x)) / (delta1 * 2)
slopes[i] = current_slope
return numpy.array(slopes)
return numeric_gradient_calculator
if __name__ == '__main__':
parameters = getData()
initial_guess = numpy.array([-65.0, -65.0])
# Parameters for negative gaussian
step_size = 100000000
threshold = 0.00001
#Setup for negative gaussian
gaussian_mean = parameters[0]
gaussian_cov = parameters[1]
objective_f = make_negative_gaussian(gaussian_mean, gaussian_cov)
gradient_f = make_negative_gaussian_derivative(objective_f, gaussian_mean,
gaussian_cov)
#Setup for quadratic bowl
