def least_squares(X, Y, order):
"""Least squares polynomial regression or arbitrary order
The equation y = a_0 + a_1*x + a_2*x^2 + ... is fitted to a series of
data points using the Vandermonde matrix and QR decomposition.
This function takes 3 arguments:
X the x values of the points to fit
Y the y values of the points to fit;
order the order of the polynomial to fit;
and returns
a the solution vector [a_0, a_1, ...]."""
nb_points = len(X)
# Build the Vandermonde matrix.
V = [[1.0] * nb_points for i in range(order + 1)]
for i_point in range(nb_points):
for i_order in range(order):
V[i_order + 1][i_point] = V[i_order][i_point] * X[i_point]
# Create vectors to store elements of the QR factorization.
diag = [0.0] * (order + 1)
perm = [0.0] * (order + 1)
# Solve by QR factorization.
QR.QR(V, diag, perm)
a = QR.QR_solve(V, diag, perm, Y)
return a
def iterate(self):
"""Do one iteration of the Levenberg-Marquardt algorithm
This method takes no arguments. It returns a single argument giving
the status of the solution. Possible values are:
IMPROVING the solution is improving;
MINIMUM_FOUND the gradient is null (or close enough);
CHI_2_IS_OK the value of chi square is acceptable;
CHI_2_CHANGE_TOO_SMALL the change in chi square is small;
ALL_PARAMETERS_ARE_STUCK all the parameters are stuck at their
limits.
When this method returns, MINIMUM_FOUND, CHI_2_IS_OK
CHI_2_CHANGE_TOO_SMALL or ALL_PARAMETERS_ARE_STUCK, the calling
function should stop the fit."""
self.iteration += 1
# Keep a copy of the old parameter values in case we need to revert
# to them.
for par in range(self.nb_par):
self.previous_a[par] = self.a[par]
# Build b.
for i in range(self.nb_points):
self.b[i] = (self.Yi[i] - self.Y[i]) / self.sigma[i]
# Determine which points are used considering the inequalities.
for i in range(self.nb_points):
if self.inequalities[i] == SMALLER and self.b[i] > 0.0:
self.use_point[i] = False
elif self.inequalities[i] == LARGER and self.b[i] < 0.0:
self.use_point[i] = False
else:
self.use_point[i] = True
# Get the derivative at this point.
self.dY = self.df(self.a, *self.f_par)
self.nb_df_eval += 1
# Calculate the gradient.
for par in range(self.nb_par):
self.beta[par] = 0.0
for i in range(self.nb_points):
if self.use_point[i]:
self.beta[par] += (self.Yi[i] - self.Y[i]) / (
self.sigma[i] * self.sigma[i]) * self.dY[par][i]
# If a parameter is stuck at one of its limits, remove it from the
# fit.
self.nb_free_par = self.nb_par
for par in range(self.nb_par):
if self.a[par] == self.a_min[par] and self.beta[par] < 0.0:
self.use_par[par] = False
self.beta[par] = 0.0
self.nb_free_par -= 1
elif self.a[par] == self.a_max[par] and self.beta[par] > 0.0:
self.use_par[par] = False
self.beta[par] = 0.0
self.nb_free_par -= 1
else:
self.use_par[par] = True
if self.nb_free_par == 0:
return ALL_PARAMETERS_ARE_STUCK
# Calculate the norm of the gradient.
self.norm_gradient = 0.0
for par in range(self.nb_par):
self.norm_gradient += self.beta[par] * self.beta[par]
self.norm_gradient = sqrt(self.norm_gradient)
# Check if the minimum is reached (norm of the gradient is 0).
if self.norm_gradient < self.min_gradient:
return MINIMUM_FOUND
# Build the jacobian matrix.
for par in range(self.nb_par):
if self.use_par[par]:
for i in range(self.nb_points):
if self.use_point[i]:
self.A[par][i] = self.dY[par][i] / self.sigma[i]
else:
self.A[par][i] = 0.0
for i in range(self.nb_points, self.nb_par):
self.A[par][i] = 0.0
else:
for i in range(self.nb_rows):
self.A[par][i] = 0.0
# Factorize the system using QR factorization and calculate inv(Q)*b.
rank = QR.QR(self.A, self.diag, self.perm, self.column_norms)
QR.QTb(self.A, self.diag, self.perm, self.b)
# On the first iteration, initialize D and the trust region to a
# fraction of the scaled length of a. And calculate the norm of a
# for a first time.
if self.iteration == 1:
for par in range(self.nb_par):
self.D[par] = self.column_norms[par]
if self.D[par] == 0.0:
self.D[par] = 1.0
norm = 0.0
for par in range(self.nb_par):
temp = self.D[par] * self.a[par]
norm += temp * temp
norm = sqrt(norm)
self.Delta = self.factor * norm
if self.Delta == 0.0:
self.Delta = self.factor
# Calculate the norm of the scaled a. This is used to check if
# Delta gets too small.
self.norm_scaled_a = 0.0
for par in range(self.nb_par):
temp = self.D[par] * self.a[par]
self.norm_scaled_a += temp * temp
self.norm_scaled_a = sqrt(self.norm_scaled_a)
# Update D if the norm of the columns has increased.
for par in range(self.nb_par):
self.D[par] = max(self.D[par], self.column_norms[par])
# Iterate until a an improving step is found.
while True:
# Compute the Gauss-Newton iteration. Here, and in the rest of
# the method, the matrix will be considerer to have full rank
# if the rank is equal to the number of free parameters. This
# works because the QR method ignores rows with null norms.
if rank == self.nb_free_par:
QR.R_solve(self.A, self.diag, self.perm, self.b, self.da)
else:
QR.rank_deficient_R_solve(self.A, self.diag, self.perm, self.b,
self.da)
# Calculate the norm of the scaled Gauss-Newton step.
norm_scaled_da = 0.0
for par in range(self.nb_par):
self.scaled_da[par] = self.D[par] * self.da[par]
norm_scaled_da += self.scaled_da[par] * self.scaled_da[par]
norm_scaled_da = sqrt(norm_scaled_da)
# If the Gauss-Newton step is accepted, set the Levenberg-
# Marquardt parameter to 0.
phi = norm_scaled_da - self.Delta
if phi <= 0.1 * self.Delta:
self.alpha = 0.0
# Otherwise, search the Levenberg-Marquardt parameter for which
# phi = 0.
else:
# Set the lower bound of alpha to -phi(0)/phi'(0). If the
# matrix is rank deficient, set the lower bound to 0.
if rank == self.nb_free_par:
for par in range(self.nb_par):
self.temp_array[par] = self.D[self.perm[par]] * (
self.scaled_da[self.perm[par]] / norm_scaled_da)
norm_square = 0.0
for par in range(self.nb_par):
if self.use_par[self.perm[par]]:
sum = 0.0
for i in range(par):
sum += self.temp_array[i] * self.A[par][i]
self.temp_array[par] = (self.temp_array[par] -
sum) / self.diag[par]
norm_square += self.temp_array[
par] * self.temp_array[par]
l = (phi / self.Delta) / norm_square
else:
l = 0.0
# Choose an upper bound. The upper bound is norm([J inv(D)]'b)
# = norm(inv(D) R' Q'b). We already have Q'b, so let's use it.
norm = 0.0
for par in range(self.nb_par):
if self.use_par[self.perm[par]]:
temp = self.diag[par] * self.b[par]
for i in range(par):
temp += self.A[par][i] * self.b[i]
temp /= self.D[self.perm[par]]
norm += temp * temp
norm = sqrt(norm)
u = norm / self.Delta
# If alpha is outside bounds, set it to the closest bound.
self.alpha = max(self.alpha, l)
self.alpha = min(self.alpha, u)
# Guess an appropriate starting value for alpha.
if self.alpha == 0.0:
self.alpha = norm / norm_scaled_da
# Search for a maximum of 10 iterations.
for internal_iteration in range(10):
# Protect ourselves against very small values of alpha (in
# particular 0).
if self.alpha == 0.0:
self.alpha = 0.001 * u
# Compute the step for the current value of alpha.
for par in range(self.nb_par):
if self.use_par[par]:
self.alpha_D[par] = sqrt(self.alpha) * self.D[par]
else:
self.alpha_D[par] = 0.0
QR.R_solve_with_update(self.A, self.diag, self.perm,
self.b, self.alpha_D, self.da)
# Calculate the norm of the scaled step.
norm_scaled_da = 0.0
for par in range(self.nb_par):
self.scaled_da[par] = self.D[par] * self.da[par]
norm_scaled_da += self.scaled_da[par] * self.scaled_da[
par]
norm_scaled_da = sqrt(norm_scaled_da)
phi = norm_scaled_da - self.Delta
# If phi is small enough, accept the step.
if abs(phi) <= 0.1 * self.Delta:
break
for par in range(self.nb_par):
if self.use_par[par]:
self.temp_array[par] = self.D[self.perm[par]] * (
self.scaled_da[self.perm[par]] /
norm_scaled_da)
# Calculate the correction.
norm_square = 0.0
for par in range(self.nb_par):
if self.use_par[self.perm[par]]:
sum = 0.0
for i in range(par):
sum += self.temp_array[i] * self.A[i][par]
self.temp_array[par] = (self.temp_array[par] -
sum) / self.A[par][par]
norm_square += self.temp_array[
par] * self.temp_array[par]
correction = (phi / self.Delta) / norm_square
# Change the bounds according to the sign of phi.
if phi > 0.0:
l = max(l, self.alpha)
else:
u = min(u, self.alpha)
self.alpha = max(self.alpha + correction, l)
# Change the parameters a by the amount suggested by da.
for par in range(self.nb_par):
self.a[par] += self.da[par]
# Check if parameters fell outside of acceptable range. Change
# both da and a since we want to be able to compare expected and
# predicted results.
bounded = False
for par in range(self.nb_par):
if self.a[par] < self.a_min[par]:
self.da[par] += self.a_min[par] - self.a[par]
self.a[par] = self.a_min[par]
bounded = True
elif self.a[par] > self.a_max[par]:
self.da[par] += self.a_max[par] - self.a[par]
self.a[par] = self.a_max[par]
bounded = True
# If one of the parameter was bounded during this iteration,
# recalculate the scaled norm of da.
if bounded:
norm_scaled_da = 0.0
for par in range(self.nb_par):
self.scaled_da[par] = self.D[par] * self.da[par]
norm_scaled_da += self.scaled_da[par] * self.scaled_da[par]
norm_scaled_da = sqrt(norm_scaled_da)
# Evaluation the function at the new point.
self.Y = self.f(self.a, *self.f_par)
self.nb_f_eval += 1
# Calculate chi_2.
new_chi_2 = 0.0
for i in range(self.nb_points):
if self.inequalities[i] == SMALLER and self.Y[i] < self.Yi[i]:
continue
elif self.inequalities[i] == LARGER and self.Y[i] > self.Yi[i]:
continue
temp = (self.Yi[i] - self.Y[i]) / self.sigma[i]
new_chi_2 += temp * temp
# Calculate the normalized actual reduction.
actual_reduction = 1.0 - (new_chi_2 / self.chi_2)
# Calculate the normalized predicted reduction of chi square and
# gamma.
part1 = 0.0
for i in range(self.nb_points):
if self.use_point[i]:
temp = 0.0
for par in range(self.nb_par):
if self.use_par[par]:
temp += self.dY[par][i] * self.da[
par] / self.sigma[i]
part1 += temp * temp
part1 /= self.chi_2
part2 = self.alpha * norm_scaled_da * norm_scaled_da / self.chi_2
predicted_reduction = part1 + 2.0 * part2
gamma = -(part1 + part2)
# Compare the actual and the predicted reduction.
rho = actual_reduction / predicted_reduction
# If the ratio is low (or negative), reduce the trust region.
if rho <= 0.25:
if actual_reduction >= 0.0:
mu = 0.5 * gamma / (gamma + 0.5 * actual_reduction)
else:
mu = 0.5
if (0.1 * new_chi_2 >= self.chi_2 or mu < 0.1):
mu = 0.1
self.Delta = mu * min(self.Delta, 10.0 * norm_scaled_da)
self.alpha /= mu
# If the ratio is high, augment the trust region.
elif rho >= 0.75 or self.alpha == 0.0:
self.Delta = 2.0 * norm_scaled_da
self.alpha *= 0.5
# If there has been improvement, accept the solution and verify if
# one of the stopping criteria is met.
if new_chi_2 < self.chi_2:
self.chi_2 = new_chi_2
# Calculate the norm of the scaled a. This is used to check if
# Delta gets too small.
self.norm_scaled_a = 0.0
for par in range(self.nb_par):
if self.use_par[par]:
temp = self.D[par] * self.a[par]
self.norm_scaled_a += temp * temp
self.norm_scaled_a = sqrt(self.norm_scaled_a)
# Verify if step criteria are met.
if self.chi_2 <= self.acceptable_chi_2:
return CHI_2_IS_OK
elif actual_reduction < self.min_chi_2_change and predicted_reduction < self.min_chi_2_change:
return CHI_2_CHANGE_TOO_SMALL
return IMPROVING
# Otherwise revert to the previous solution and try again.
else:
for par in range(self.nb_par):
self.a[par] = self.previous_a[par]
# If Delta is smaller than the machine precision, we cannot do
# any better!
if self.norm_scaled_a == 0.0:
if self.Delta < epsilon:
return DELTA_IS_TOO_SMALL
else:
if self.Delta / self.norm_scaled_a < epsilon:
return DELTA_IS_TOO_SMALL
import Hess
import QR
import numpy as np
import scipy
from scipy import linalg
# A = [[2.,-1.,0.,0.],[-1.,2.,-1.,0.],[0.,-1.,2.,-1.],[0.,0.,-1.,2.]]
A = np.random.rand(50, 5)
# b = [0.,5.,5.,5.]
# test = Hess.hess(A)
# np.set_printoptions(precision=15)
# for row in test:
# for elem in row:
# print('{:.12e}'.format(elem))
# print('\n')
Q, R, err = QR.QR(A)
# print(Q)
print('Error in QR factorization = ' + str(err))
# x = QR.QRsolve(A,b)
# print(x)
import numpy as np
import Jacobi
import QR
np.set_printoptions(precision=3)
try:
A = np.loadtxt("matrix.txt")
except (Exception):
print("Could not find file or invalid matrix")
exit(-1)
print("Our matrix:")
print(A)
# try:
# L, X = Jacobi.Jacobi(A)
# except Exception as e:
# print(e)
# exit(-1)
# print("Eigenvalues:\n", np.array2string(L))
# print("Eigenvectors:\n", np.array2string(X))
# print("Q * Qt =\n", np.array2string(X @ X.T))
# print("Q * L * Qt =\n", np.array2string(X @ L @ X.T))
# A = np.random.random(16).reshape(4, 4)
R = QR.QR(A)
print("Result matrix:\n" + np.array2string(R))
print("Answer via numpy: " + np.array2string(np.linalg.eigvals(A)))
from QR import *
a = QR('HELLO WORLD', True)
a.modo_correcao = Q
a.versao = 1
a.analise()
a.codificar()
print(a)
# 00100000 01011011 00001011 01111000 11010001 01110010 11011100 01001101 01000011 01000000 11101100 00010001 11101100
# 00100000 01011011 00001011 01111000 11010001 01110010 11011100 01001101 01000011 01000000 11101100 00010001
# 00100000 01011011 00001011 01111000 11010001 01110010 11011100 01001101 01000011 01000000 11101100 00010001 11101100
