def line_search_for_residual_by_backtracking(r_primal,
r_dual,
dir_desc_primal,
dir_desc_dual,
x,
nu,
norm_residual_eval,
alpha=.15,
beta=.5):
'''
Busqueda de linea que disminuye suficientemente el residual para el metodo
de punto inicial inviable de Newton restringido a un rayo en la direccion dir_desc.
Args:
r_primal (fun): definicion de residual primal como funcion definida o expresion lambda.
r_dual (fun): definicion de residual dual como funcion definida o expresion lambda.
dir_desc_primal (array): direccion de desceso para variable primal.
dir_desc_dual (array): direccion de descenso para variable dual.
x (array): array de cupy que contiene valores donde se realizara la busqueda de linea.
nu (array): array de cupy que contiene valores donde se realizara la busqueda de linea.
norm_residual_eval (float): norma de residuos que tiene evaluaciones r_primal y
r_dual en x y nu
alpha (float): parametro de busqueda de linea con backtracking, tipicamente .15
beta (float): parametro de busqueda de linea con backtracking, tipicamente .5
Returns:
t (float): numero positivo para el tamano de pasos a lo largo de dir_desc que
disminuye suficientemente f.
'''
t = 1
if alpha > 1 / 2:
print('alpha debe ser menor o igual que 1/2')
t = -1
if beta > 1:
print('beta debe ser menor que uno 1')
t = -1
if t != -1:
feas_primal = r_primal(x + t * dir_desc_primal)
feas_dual = r_dual(nu + t * dir_desc_dual)
eval1 = norm_residual(feas_primal, feas_dual)
eval2 = (1 - alpha * t) * norm_residual_eval
while eval1 > eval2:
t = beta * t
feas_primal = r_primal(x + t * dir_desc_primal)
feas_dual = r_dual(nu + t * dir_desc_dual)
eval1 = norm_residual(feas_primal, feas_dual)
eval2 = (1 - alpha * t) * norm_residual_eval
return t
def line_search_for_residual_by_backtracking(r_primal,
r_dual,
dir_desc_primal,
dir_desc_dual,
x,
nu,
norm_residual_eval,
alpha=.15,
beta=.5):
'''
Line search that sufficiently decreases residual for Newtons infeasible initial point method
restricted to a ray in the direction dir_desc.
Args:
r_primal (fun): definition of primal residual as function definition or lambda expression.
r_dual (fun): definition of dual residual as function definition or lambda expression.
dir_desc_primal (array): descent direction for primal variable.
dir_desc_dual (array): descent direction for dual variable.
x (array): numpy array that holds values where line search will be performed.
nu (array): numpy array that holds values where line search will be performed.
norm_residual_eval (float): norm of residual that has both r_primal and r_dual evaluations in
x and nu
alpha (float): parameter in line search with backtracking, tipically .15
beta (float): parameter in line search with backtracking, tipically .5
Returns:
t (float): positive number for stepsize along dir_desc that sufficiently decreases f.
'''
t = 1
if alpha > 1 / 2:
print('alpha must be less than or equal to 1/2')
t = -1
if beta > 1:
print('beta must be less than 1')
t = -1
if t != -1:
feas_primal = r_primal(x + t * dir_desc_primal)
feas_dual = r_dual(nu + t * dir_desc_dual)
eval1 = norm_residual(feas_primal, feas_dual)
eval2 = (1 - alpha * t) * norm_residual_eval
while eval1 > eval2:
t = beta * t
feas_primal = r_primal(x + t * dir_desc_primal)
feas_dual = r_dual(nu + t * dir_desc_dual)
eval1 = norm_residual(feas_primal, feas_dual)
eval2 = (1 - alpha * t) * norm_residual_eval
return t
def Newtons_method_log_barrier_infeasible_init_point(f, A, b,
constraint_inequalities, t_path,
x_0, nu_0, tol,
tol_backtracking, x_ast=None, p_ast=None, maxiter=30,
gf_symbolic = None,
Hf_symbolic = None):
'''
Newton's method for infeasible initial point to numerically approximate solution of min f subject to Ax = b.
Calls Newton's method for feasible initial point when reach primal feasibility given by tol.
Args:
f (fun): definition of function f as lambda expression or function definition.
A (numpy ndarray): 2d numpy array of shape (m,n) defines system of constraints Ax=b.
b (numpy ndarray or float): array that defines system of constraints Ax=b (can be a single number
if just one restriction is defined)
constraints_ineq
t_path
x_0 (numpy ndarray): initial point for Newton's method.
nu_0 (numpy ndarray or float): initial point for Newton's method.
tol (float): tolerance that will halt method. Controls stopping criteria.
tol_backtracking (float): tolerance that will halt method. Controls value of line search by backtracking.
x_ast (numpy ndarray): solution of min f, now it's required that user knows the solution...
p_ast (float): value of f(x_ast), now it's required that user knows the solution...
maxiter (int): maximum number of iterations
gf_symbolic (fun): definition of gradient of f. If given, no approximation is
performed via finite differences.
Hf_symbolic (fun): definition of Hessian of f. If given, no approximation is
performed via finite differences.
Returns:
x (numpy ndarray): numpy array, approximation of x_ast.
iteration (int): number of iterations.
Err_plot (numpy ndarray): numpy array of absolute error between p_ast and f(x) with x approximation
of x_ast. Useful for plotting.
x_plot (numpy ndarray): numpy array that containts in columns vector of approximations. Last column
contains x, approximation of solution. Useful for plotting.
'''
iteration = 0
x = x_0
nu = nu_0
feval = f(x)
if gf_symbolic and Hf_symbolic:
gfeval = gf_symbolic(x)
Hfeval = Hf_symbolic(x)
else:
grad_Hess_log_barrier_dict = numerical_differentiation_of_logarithmic_barrier(f,x,t_path,
constraint_inequalities)
gfeval = grad_Hess_log_barrier_dict['gradient']
Hfeval = grad_Hess_log_barrier_dict['Hessian']
normgf = np.linalg.norm(gfeval)
condHf= np.linalg.cond(Hfeval)
Err_x_ast = compute_error(x_ast,x)
Err_p_ast = compute_error(p_ast,feval)
def residual_dual(nu_fun):
if(A.ndim==1):
return gfeval + A.T*nu_fun
else:
return gfeval + A.T@nu_fun
feasibility_dual = residual_dual(nu)
if (np.all(A < np.nextafter(0,1))): #A is zero matrix
system_matrix = Hfeval
rhs = -gfeval
n = x.size
p = 0
feasibility_primal = 0
else:
if(A.ndim == 1):
p = 1
n = x.size
zero_matrix = np.zeros(p)
first_stack = np.column_stack((Hfeval, A.T))
second_stack = np.row_stack((A.reshape(1,n).T,zero_matrix)).reshape(1,n+1)[0]
else:
p,n = A.shape
zero_matrix = np.zeros((p,p))
first_stack = np.column_stack((Hfeval, A.T))
second_stack = np.column_stack((A,zero_matrix))
system_matrix = np.row_stack((first_stack,second_stack))
residual_primal = lambda x_fun: A@x_fun-b
feasibility_primal = residual_primal(x)
rhs = -np.row_stack((feasibility_dual.reshape(n,1), feasibility_primal.reshape(p,1))).T[0]
#Newton's direction and Newton's decrement
dir_desc = np.linalg.solve(system_matrix, rhs)
dir_Newton_primal = dir_desc[0:n]
dec_Newton = -gfeval.dot(dir_Newton_primal)
dir_Newton_dual = dir_desc[n:(n+p)]
norm_residual_eval = norm_residual(feasibility_primal,
feasibility_dual)
norm_residual_primal = np.linalg.norm(feasibility_primal)
norm_residual_dual = np.linalg.norm(feasibility_dual)
print('I\t||res_primal||\t||res_dual|| \tNewton Decrement\tError x_ast\tError p_ast\tline search\tCondHf')
print('{}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{}\t\t{:0.2e}'.format(iteration,norm_residual_primal,
norm_residual_dual,
dec_Newton,Err_x_ast,
Err_p_ast,"---",
condHf))
stopping_criteria = norm_residual_primal > tol
iteration+=1
while(stopping_criteria>tol and iteration < maxiter):
der_direct = -dec_Newton
t = line_search_for_residual_by_backtracking(residual_primal, residual_dual,
dir_Newton_primal, dir_Newton_dual,
x, nu,
norm_residual_eval
)
x = x + t*dir_Newton_primal
nu = nu + t*dir_Newton_dual
feval = f(x)
if gf_symbolic:
gfeval = gf_symbolic(x)
else:
gfeval = gradient_approximation(f,x)
if Hf_symbolic:
Hfeval = Hf_symbolic(x)
else:
Hfeval = Hessian_approximation(f,x)
if(A.ndim == 1):
first_stack = np.column_stack((Hfeval, A.T))
else:
first_stack = np.column_stack((Hfeval, A.T))
system_matrix = np.row_stack((first_stack,second_stack))
feasibility_primal = residual_primal(x)
feasibility_dual = residual_dual(nu)
rhs = -np.row_stack((feasibility_dual.reshape(n,1), feasibility_primal.reshape(p,1))).T[0]
#Newton's direction and Newton's decrement
dir_desc = np.linalg.solve(system_matrix, rhs)
dir_Newton_primal = dir_desc[0:n]
dec_Newton = -gfeval.dot(dir_Newton_primal)
dir_Newton_dual = dir_desc[n:(n+p)]
Err_x_ast = compute_error(x_ast,x)
Err_p_ast = compute_error(p_ast,feval)
norm_residual_eval = norm_residual(feasibility_primal,
feasibility_dual)
norm_residual_primal = np.linalg.norm(feasibility_primal)
norm_residual_dual = np.linalg.norm(feasibility_dual)
print('{}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}'.format(iteration,norm_residual_primal,
norm_residual_dual,
dec_Newton,Err_x_ast,
Err_p_ast,t,
condHf))
stopping_criteria = norm_residual_primal > tol
if t<tol_backtracking: #if t is less than tol_backtracking then we need to check the reason
iteration = maxiter - 1
iteration+=1
if iteration == maxiter and t < tol_backtracking:
print("Backtracking value less than tol_backtracking, try other initial point")
return [None,None,None,None]
else:
if iteration == maxiter:
print("------------------------------------------------------------")
print("------------------------------------------------------------")
print("------------------------------------------------------------")
print("Reached maximum of iterations, check primal feasibility")
print("Continuing with Newtons method for feasible initial point")
return Newtons_method_log_barrier_feasible_init_point(f,A, constraint_inequalities,
t_path, x,tol,
tol_backtracking, x_ast, p_ast,
maxiter,gf_symbolic,Hf_symbolic)
else:
print("------------------------------------------------------------")
print("------------------------------------------------------------")
print("------------------------------------------------------------")
print("Beginning Newtons method for feasible initial point")
return Newtons_method_log_barrier_feasible_init_point(f,A, constraint_inequalities,
t_path, x,tol,
tol_backtracking, x_ast, p_ast,
maxiter,gf_symbolic,Hf_symbolic)
def Newtons_method_infeasible_init_point_2nd_version(f,
A,
b,
x_0,
nu_0,
tol,
tol_backtracking,
x_ast=None,
p_ast=None,
maxiter=30,
gf_symbolic=None,
Hf_symbolic=None):
'''
Newton's method to numerically approximate solution of min f subject to Ax = b.
Args:
f (fun): definition of function f as lambda expression or function definition.
A (numpy ndarray): 2d numpy array of shape (m,n) defines system of constraints Ax=b.
b (numpy ndarray or float): array that defines system of constraints Ax=b (can be a single number
if just one restriction is defined)
x_0 (numpy ndarray): initial point for Newton's method.
nu_0 (numpy ndarray): initial point for Newton's method.
tol (float): tolerance that will halt method. Controls stopping criteria.
tol_backtracking (float): tolerance that will halt method. Controls value of line search by backtracking.
x_ast (numpy ndarray): solution of min f, now it's required that user knows the solution...
p_ast (float): value of f(x_ast), now it's required that user knows the solution...
maxiter (int): maximum number of iterations
gf_symbolic (fun): definition of gradient of f. If given, no approximation is
performed via finite differences.
Hf_symbolic (fun): definition of Hessian of f. If given, no approximation is
performed via finite differences.
Returns:
x (numpy ndarray): numpy array, approximation of x_ast.
iteration (int): number of iterations.
Err_plot (numpy ndarray): numpy array of absolute error between p_ast and f(x) with x approximation
of x_ast. Useful for plotting.
x_plot (numpy ndarray): numpy array that containts in columns vector of approximations. Last column
contains x, approximation of solution. Useful for plotting.
'''
iteration = 0
x = x_0
nu = nu_0
feval = f(x)
if gf_symbolic:
gfeval = gf_symbolic(x)
else:
gfeval = gradient_approximation(f, x)
if Hf_symbolic:
Hfeval = Hf_symbolic(x)
else:
Hfeval = Hessian_approximation(f, x)
normgf = np.linalg.norm(gfeval)
condHf = np.linalg.cond(Hfeval)
Err_plot_aux = np.zeros(maxiter)
Err_plot_aux[iteration] = compute_error(p_ast, feval)
Err = compute_error(x_ast, x)
if (A.ndim == 1):
p = 1
n = x.size
zero_matrix = np.zeros(p)
first_stack = np.column_stack((Hfeval, A.T))
second_stack = np.row_stack(
(A.reshape(1, n).T, zero_matrix)).reshape(1, n + 1)[0]
else:
p, n = A.shape
zero_matrix = np.zeros((p, p))
first_stack = np.column_stack((Hfeval, A.T))
second_stack = np.column_stack((A, zero_matrix))
x_plot = np.zeros((n, maxiter))
x_plot[:, iteration] = x
system_matrix = np.row_stack((first_stack, second_stack))
residual_primal = lambda x_fun: A @ x_fun - b
def residual_dual(nu_fun):
if (A.ndim == 1):
return gfeval + A.T * nu_fun
else:
return gfeval + A.T @ nu_fun
feasibility_primal = residual_primal(x)
feasibility_dual = residual_dual(nu)
rhs = np.row_stack(
(feasibility_dual.reshape(n, 1), feasibility_primal.reshape(p,
1))).T[0]
#Newton's direction and Newton's decrement
dir_desc = np.linalg.solve(system_matrix, -rhs)
dir_Newton_primal = dir_desc[0:n]
dec_Newton = -gfeval.dot(dir_Newton_primal)
dir_Newton_dual = dir_desc[n:(n + p)]
norm_residual_eval = norm_residual(feasibility_primal, feasibility_dual)
norm_residual_primal = np.linalg.norm(feasibility_primal)
norm_residual_dual = np.linalg.norm(feasibility_dual)
print(
'I\t||res_primal||\t||res_dual|| \tNewton Decrement\tError x_ast\tError p_ast\tline search\tCondHf'
)
print('{}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{}\t\t{:0.2e}'.
format(iteration, norm_residual_primal, norm_residual_dual,
dec_Newton, Err, Err_plot_aux[iteration], "---", condHf))
stopping_criteria = norm_residual_eval > tol
iteration += 1
while (stopping_criteria > tol and iteration < maxiter):
der_direct = -dec_Newton
t = line_search_for_residual_by_backtracking(residual_primal,
residual_dual,
dir_Newton_primal,
dir_Newton_dual, x, nu,
norm_residual_eval)
x = x + t * dir_Newton_primal
nu = nu + t * dir_Newton_dual
feval = f(x)
if gf_symbolic:
gfeval = gf_symbolic(x)
else:
gfeval = gradient_approximation(f, x)
if Hf_symbolic:
Hfeval = Hf_symbolic(x)
else:
Hfeval = Hessian_approximation(f, x)
if (A.ndim == 1):
first_stack = np.column_stack((Hfeval, A.T))
else:
first_stack = np.column_stack((Hfeval, A.T))
system_matrix = np.row_stack((first_stack, second_stack))
feasibility_primal = residual_primal(x)
feasibility_dual = residual_dual(nu)
rhs = np.row_stack((feasibility_dual.reshape(n, 1),
feasibility_primal.reshape(p, 1))).T[0]
#Newton's direction and Newton's decrement
dir_desc = np.linalg.solve(system_matrix, -rhs)
dir_Newton_primal = dir_desc[0:n]
dec_Newton = -gfeval.dot(dir_Newton_primal)
dir_Newton_dual = dir_desc[n:(n + p)]
Err_plot_aux[iteration] = compute_error(p_ast, feval)
x_plot[:, iteration] = x
Err = compute_error(x_ast, x)
norm_residual_eval = norm_residual(feasibility_primal,
feasibility_dual)
norm_residual_primal = np.linalg.norm(feasibility_primal)
norm_residual_dual = np.linalg.norm(feasibility_dual)
print(
'{}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}'
.format(iteration, norm_residual_primal, norm_residual_dual,
dec_Newton, Err, Err_plot_aux[iteration], t, condHf))
stopping_criteria = norm_residual_eval > tol
if t < tol_backtracking: #if t is less than tol_backtracking then we need to check the reason
iter_salida = iteration
iteration = maxiter - 1
iteration += 1
print('{} {:0.2e}'.format("Error of x with respect to x_ast:", Err))
print('{} {}'.format("Approximate solution:", x))
cond = Err_plot_aux > np.finfo(float).eps * 10**(-2)
Err_plot = Err_plot_aux[cond]
if iteration == maxiter and t < tol_backtracking:
print(
"Backtracking value less than tol_backtracking, check approximation"
)
iteration = iter_salida
else:
if iteration == maxiter:
print("Reached maximum of iterations, check approximation")
x_plot = x_plot[:, :iteration]
return [x, iteration, Err_plot, x_plot]
