def lstsq_regression(cls, matrix_a, matrix_b, intercept=False):
"""
Performs Least Squares Regression.
Solves the problem:
:math:`X = argmin(||AX - B||_2)`
Args:
matrix_a: input matrix A, of type Matrix
matrix_b: input matrix A, of type Matrix
intercept: bool. If True intercept is used. Optional, False by default.
Returns:
solution X of type Matrix
"""
matrix_a._assert_same_type(matrix_b)
# TODO: check out where to define this assert
assert_same_shape(matrix_a, matrix_b, 0)
if intercept:
matrix_a = matrix_a.hstack(type(matrix_a)(np.ones((matrix_a.shape[0],
1))))
if isinstance(matrix_a, DenseMatrix):
result = Linalg._dense_lstsq_regression(matrix_a, matrix_b)
else:
result = Linalg._sparse_lstsq_regression(matrix_a, matrix_b)
return result
def lstsq_regression(cls, matrix_a, matrix_b, intercept=False):
"""
Performs Least Squares Regression.
Solves the problem:
:math:`X = argmin(||AX - B||_2)`
Args:
matrix_a: input matrix A, of type Matrix
matrix_b: input matrix A, of type Matrix
intercept: bool. If True intercept is used. Optional, False by default.
Returns:
solution X of type Matrix
"""
matrix_a._assert_same_type(matrix_b)
# TODO: check out where to define this assert
assert_same_shape(matrix_a, matrix_b, 0)
if intercept:
matrix_a = matrix_a.hstack(
type(matrix_a)(np.ones((matrix_a.shape[0], 1))))
if isinstance(matrix_a, DenseMatrix):
result = Linalg._dense_lstsq_regression(matrix_a, matrix_b)
else:
result = Linalg._sparse_lstsq_regression(matrix_a, matrix_b)
return result
def ridge_regression(matrix_a , matrix_b, lambda_, intercept=False):
#log.print_info(logger, "In Ridge regression..", 4)
#log.print_matrix_info(logger, matrix_a, 5, "Input matrix A:")
#log.print_matrix_info(logger, matrix_b, 5, "Input matrix B:")
"""
Performs Ridge Regression.
This method use the general formula:
...
to solve the problem:
:math:`X = argmin(||AX - B||_2 + \\lambda||X||_2)`
Args:
matrix_a: input matrix A, of type Matrix
matrix_b: input matrix A, of type Matrix
lambda_: scalar, lambda parameter
intercept: bool. If True intercept is used. Optional, default False.
Returns:
solution X of type Matrix
"""
matrix_a._assert_same_type(matrix_b)
# TODO: check out where to define this assert
assert_same_shape(matrix_a, matrix_b, 0)
matrix_type = type(matrix_a)
dim = matrix_a.shape[1]
if intercept:
matrix_a = matrix_a.hstack(matrix_type(np.ones((matrix_a.shape[0],
1))))
lambda_diag = (lambda_ ) * matrix_type.identity(dim)
if intercept:
lambda_diag = padd_matrix(padd_matrix(lambda_diag, 0, 0.0), 1, 0.0)
matrix_a_t = matrix_a.transpose()
try:
tmp_mat = Linalg.pinv(((matrix_a_t * matrix_a) + lambda_diag))
except np.linalg.LinAlgError:
print "Warning! LinAlgError"
tmp_mat = matrix_type.identity(lambda_diag.shape[0])
tmp_res = tmp_mat * matrix_a_t
result = tmp_res * matrix_b
#S: used in generalized cross validation, page 244 7.52 (YZ also used it)
# S is defined in 7.31, page 232
# instead of computing the matrix and then its trace, we can compute
# its trace directly
# NOTE when lambda = 0 we get out trace(S) = rank(matrix_a)
dist = (matrix_a * result - matrix_b).norm()
S_trace = matrix_a_t.multiply(tmp_res).sum()
return result, S_trace, dist
def ridge_regression(matrix_a, matrix_b, lambda_, intercept=False):
#log.print_info(logger, "In Ridge regression..", 4)
#log.print_matrix_info(logger, matrix_a, 5, "Input matrix A:")
#log.print_matrix_info(logger, matrix_b, 5, "Input matrix B:")
"""
Performs Ridge Regression.
This method use the general formula:
...
to solve the problem:
:math:`X = argmin(||AX - B||_2 + \\lambda||X||_2)`
Args:
matrix_a: input matrix A, of type Matrix
matrix_b: input matrix A, of type Matrix
lambda_: scalar, lambda parameter
intercept: bool. If True intercept is used. Optional, default False.
Returns:
solution X of type Matrix
"""
matrix_a._assert_same_type(matrix_b)
# TODO: check out where to define this assert
assert_same_shape(matrix_a, matrix_b, 0)
matrix_type = type(matrix_a)
dim = matrix_a.shape[1]
if intercept:
matrix_a = matrix_a.hstack(
matrix_type(np.ones((matrix_a.shape[0], 1))))
lambda_diag = (lambda_) * matrix_type.identity(dim)
if intercept:
lambda_diag = padd_matrix(padd_matrix(lambda_diag, 0, 0.0), 1, 0.0)
matrix_a_t = matrix_a.transpose()
try:
tmp_mat = Linalg.pinv(((matrix_a_t * matrix_a) + lambda_diag))
except np.linalg.LinAlgError:
print "Warning! LinAlgError"
tmp_mat = matrix_type.identity(lambda_diag.shape[0])
tmp_res = tmp_mat * matrix_a_t
result = tmp_res * matrix_b
#S: used in generalized cross validation, page 244 7.52 (YZ also used it)
# S is defined in 7.31, page 232
# instead of computing the matrix and then its trace, we can compute
# its trace directly
# NOTE when lambda = 0 we get out trace(S) = rank(matrix_a)
dist = (matrix_a * result - matrix_b).norm()
S_trace = matrix_a_t.multiply(tmp_res).sum()
return result, S_trace, dist
def tracenorm_regression(matrix_a , matrix_b, lmbd, iterations, intercept=False):
#log.print_info(logger, "In Tracenorm regression..", 4)
#log.print_matrix_info(logger, matrix_a, 5, "Input matrix A:")
#log.print_matrix_info(logger, matrix_b, 5, "Input matrix B:")
"""
Performs Trace Norm Regression.
This method uses approximate gradient descent
to solve the problem:
:math:`X = argmin(||AX - B||_2 + \\lambda||X||_*)`
where :math:`||X||_*` is the trace norm of :math:`X`, the sum of its
singular values.
It is implemented for dense matrices only.
The algorithm is the Extended Gradient Algorithm from (Ji and Ye, 2009).
Args:
matrix_a: input matrix A, of type Matrix
matrix_b: input matrix A, of type Matrix. If None, it is defined as matrix_a
lambda_: scalar, lambda parameter
intercept: bool. If True intercept is used. Optional, default False.
Returns:
solution X of type Matrix
"""
if intercept:
matrix_a = matrix_a.hstack(matrix_type(np.ones((matrix_a.shape[0],
1))))
if matrix_b == None:
matrix_b = matrix_a
# TODO remove this
matrix_a = DenseMatrix(matrix_a).mat
matrix_b = DenseMatrix(matrix_b).mat
# Matrix shapes
p = matrix_a.shape[0]
q = matrix_a.shape[1]
assert_same_shape(matrix_a, matrix_b, 0)
# Initialization of the algorithm
W = (1.0/p)* Linalg._kronecker_product(matrix_a)
# Sub-expressions reused at various places in the code
matrix_a_t = matrix_a.transpose()
at_times_a = np.dot(matrix_a_t, matrix_a)
# Epsilon: to ensure that our bound on the Lipschitz constant is large enough
epsilon_lbound = 0.05
# Expression of the bound of the Lipschitz constant of the cost function
L_bound = (1+epsilon_lbound)*2*Linalg._frobenius_norm_squared(at_times_a)
# Current "guess" of the local Lipschitz constant
L = 1.0
# Factor by which L should be increased when it happens to be too small
gamma = 1.2
# Epsilon to ensure that mu is increased when the inequality hold tightly
epsilon_cost = 0.00001
# Real lambda: resized according to the number of training samples (?)
lambda_ = lmbd*p
# Variables used for the accelerated algorithm (check the original paper)
Z = W
alpha = 1.0
# Halting condition
epsilon = 0.00001
last_cost = 1
current_cost = -1
linalg_error_caught = False
costs = []
iter_counter = 0
while iter_counter < iterations and (abs((current_cost - last_cost)/last_cost)>epsilon) and not linalg_error_caught:
sys.stdout.flush()
# Cost tracking
try:
next_W, tracenorm = Linalg._next_tracenorm_guess(matrix_a, matrix_b, lambda_, L, Z, at_times_a)
except LinAlgError:
print "LinAlgError caught in trace norm regression"
linalg_error_caught = True
break
last_cost = current_cost
current_fitness = Linalg._fitness(matrix_a, matrix_b, next_W)
current_cost = current_fitness + lambda_ * tracenorm
if iter_counter > 0: # The first scores are messy
cost_list = [L, L_bound, current_fitness, current_cost]
costs.append(cost_list)
while (current_fitness + epsilon_cost >=
Linalg._intermediate_cost(matrix_a, matrix_b, next_W, Z, L)):
if L > L_bound:
print "Trace Norm Regression: numerical error detected at iteration "+str(iter_counter)
break
L = gamma * L
try:
next_W, tracenorm = Linalg._next_tracenorm_guess(matrix_a, matrix_b, lambda_, L, Z, at_times_a)
except LinAlgError:
print "LinAlgError caught in trace norm regression"
linalg_error_caught = True
break
last_cost = current_cost
current_fitness = Linalg._fitness(matrix_a, matrix_a, next_W)
current_cost = current_fitness + lambda_*tracenorm
if linalg_error_caught:
break
previous_W = W
W = next_W
previous_alpha = alpha
alpha = (1.0 + sqrt(1.0 + 4.0*alpha*alpha))/2.0
Z = W
# Z = W + ((alpha - 1)/alpha)*(W - previous_W)
iter_counter += 1
sys.stdout.flush()
W = np.real(W)
return DenseMatrix(W), costs
