def lstsq_regression(cls, matrix_a, matrix_b, intercept=False): """ Performs Least Squares Regression. Solves the problem: :math:`X = argmin(||AX - B||_2)` Args: matrix_a: input matrix A, of type Matrix matrix_b: input matrix A, of type Matrix intercept: bool. If True intercept is used. Optional, False by default. Returns: solution X of type Matrix """ matrix_a._assert_same_type(matrix_b) # TODO: check out where to define this assert assert_same_shape(matrix_a, matrix_b, 0) if intercept: matrix_a = matrix_a.hstack(type(matrix_a)(np.ones((matrix_a.shape[0], 1)))) if isinstance(matrix_a, DenseMatrix): result = Linalg._dense_lstsq_regression(matrix_a, matrix_b) else: result = Linalg._sparse_lstsq_regression(matrix_a, matrix_b) return result
def lstsq_regression(cls, matrix_a, matrix_b, intercept=False): """ Performs Least Squares Regression. Solves the problem: :math:`X = argmin(||AX - B||_2)` Args: matrix_a: input matrix A, of type Matrix matrix_b: input matrix A, of type Matrix intercept: bool. If True intercept is used. Optional, False by default. Returns: solution X of type Matrix """ matrix_a._assert_same_type(matrix_b) # TODO: check out where to define this assert assert_same_shape(matrix_a, matrix_b, 0) if intercept: matrix_a = matrix_a.hstack( type(matrix_a)(np.ones((matrix_a.shape[0], 1)))) if isinstance(matrix_a, DenseMatrix): result = Linalg._dense_lstsq_regression(matrix_a, matrix_b) else: result = Linalg._sparse_lstsq_regression(matrix_a, matrix_b) return result
def ridge_regression(matrix_a , matrix_b, lambda_, intercept=False): #log.print_info(logger, "In Ridge regression..", 4) #log.print_matrix_info(logger, matrix_a, 5, "Input matrix A:") #log.print_matrix_info(logger, matrix_b, 5, "Input matrix B:") """ Performs Ridge Regression. This method use the general formula: ... to solve the problem: :math:`X = argmin(||AX - B||_2 + \\lambda||X||_2)` Args: matrix_a: input matrix A, of type Matrix matrix_b: input matrix A, of type Matrix lambda_: scalar, lambda parameter intercept: bool. If True intercept is used. Optional, default False. Returns: solution X of type Matrix """ matrix_a._assert_same_type(matrix_b) # TODO: check out where to define this assert assert_same_shape(matrix_a, matrix_b, 0) matrix_type = type(matrix_a) dim = matrix_a.shape[1] if intercept: matrix_a = matrix_a.hstack(matrix_type(np.ones((matrix_a.shape[0], 1)))) lambda_diag = (lambda_ ) * matrix_type.identity(dim) if intercept: lambda_diag = padd_matrix(padd_matrix(lambda_diag, 0, 0.0), 1, 0.0) matrix_a_t = matrix_a.transpose() try: tmp_mat = Linalg.pinv(((matrix_a_t * matrix_a) + lambda_diag)) except np.linalg.LinAlgError: print "Warning! LinAlgError" tmp_mat = matrix_type.identity(lambda_diag.shape[0]) tmp_res = tmp_mat * matrix_a_t result = tmp_res * matrix_b #S: used in generalized cross validation, page 244 7.52 (YZ also used it) # S is defined in 7.31, page 232 # instead of computing the matrix and then its trace, we can compute # its trace directly # NOTE when lambda = 0 we get out trace(S) = rank(matrix_a) dist = (matrix_a * result - matrix_b).norm() S_trace = matrix_a_t.multiply(tmp_res).sum() return result, S_trace, dist
def ridge_regression(matrix_a, matrix_b, lambda_, intercept=False): #log.print_info(logger, "In Ridge regression..", 4) #log.print_matrix_info(logger, matrix_a, 5, "Input matrix A:") #log.print_matrix_info(logger, matrix_b, 5, "Input matrix B:") """ Performs Ridge Regression. This method use the general formula: ... to solve the problem: :math:`X = argmin(||AX - B||_2 + \\lambda||X||_2)` Args: matrix_a: input matrix A, of type Matrix matrix_b: input matrix A, of type Matrix lambda_: scalar, lambda parameter intercept: bool. If True intercept is used. Optional, default False. Returns: solution X of type Matrix """ matrix_a._assert_same_type(matrix_b) # TODO: check out where to define this assert assert_same_shape(matrix_a, matrix_b, 0) matrix_type = type(matrix_a) dim = matrix_a.shape[1] if intercept: matrix_a = matrix_a.hstack( matrix_type(np.ones((matrix_a.shape[0], 1)))) lambda_diag = (lambda_) * matrix_type.identity(dim) if intercept: lambda_diag = padd_matrix(padd_matrix(lambda_diag, 0, 0.0), 1, 0.0) matrix_a_t = matrix_a.transpose() try: tmp_mat = Linalg.pinv(((matrix_a_t * matrix_a) + lambda_diag)) except np.linalg.LinAlgError: print "Warning! LinAlgError" tmp_mat = matrix_type.identity(lambda_diag.shape[0]) tmp_res = tmp_mat * matrix_a_t result = tmp_res * matrix_b #S: used in generalized cross validation, page 244 7.52 (YZ also used it) # S is defined in 7.31, page 232 # instead of computing the matrix and then its trace, we can compute # its trace directly # NOTE when lambda = 0 we get out trace(S) = rank(matrix_a) dist = (matrix_a * result - matrix_b).norm() S_trace = matrix_a_t.multiply(tmp_res).sum() return result, S_trace, dist
def tracenorm_regression(matrix_a , matrix_b, lmbd, iterations, intercept=False): #log.print_info(logger, "In Tracenorm regression..", 4) #log.print_matrix_info(logger, matrix_a, 5, "Input matrix A:") #log.print_matrix_info(logger, matrix_b, 5, "Input matrix B:") """ Performs Trace Norm Regression. This method uses approximate gradient descent to solve the problem: :math:`X = argmin(||AX - B||_2 + \\lambda||X||_*)` where :math:`||X||_*` is the trace norm of :math:`X`, the sum of its singular values. It is implemented for dense matrices only. The algorithm is the Extended Gradient Algorithm from (Ji and Ye, 2009). Args: matrix_a: input matrix A, of type Matrix matrix_b: input matrix A, of type Matrix. If None, it is defined as matrix_a lambda_: scalar, lambda parameter intercept: bool. If True intercept is used. Optional, default False. Returns: solution X of type Matrix """ if intercept: matrix_a = matrix_a.hstack(matrix_type(np.ones((matrix_a.shape[0], 1)))) if matrix_b == None: matrix_b = matrix_a # TODO remove this matrix_a = DenseMatrix(matrix_a).mat matrix_b = DenseMatrix(matrix_b).mat # Matrix shapes p = matrix_a.shape[0] q = matrix_a.shape[1] assert_same_shape(matrix_a, matrix_b, 0) # Initialization of the algorithm W = (1.0/p)* Linalg._kronecker_product(matrix_a) # Sub-expressions reused at various places in the code matrix_a_t = matrix_a.transpose() at_times_a = np.dot(matrix_a_t, matrix_a) # Epsilon: to ensure that our bound on the Lipschitz constant is large enough epsilon_lbound = 0.05 # Expression of the bound of the Lipschitz constant of the cost function L_bound = (1+epsilon_lbound)*2*Linalg._frobenius_norm_squared(at_times_a) # Current "guess" of the local Lipschitz constant L = 1.0 # Factor by which L should be increased when it happens to be too small gamma = 1.2 # Epsilon to ensure that mu is increased when the inequality hold tightly epsilon_cost = 0.00001 # Real lambda: resized according to the number of training samples (?) lambda_ = lmbd*p # Variables used for the accelerated algorithm (check the original paper) Z = W alpha = 1.0 # Halting condition epsilon = 0.00001 last_cost = 1 current_cost = -1 linalg_error_caught = False costs = [] iter_counter = 0 while iter_counter < iterations and (abs((current_cost - last_cost)/last_cost)>epsilon) and not linalg_error_caught: sys.stdout.flush() # Cost tracking try: next_W, tracenorm = Linalg._next_tracenorm_guess(matrix_a, matrix_b, lambda_, L, Z, at_times_a) except LinAlgError: print "LinAlgError caught in trace norm regression" linalg_error_caught = True break last_cost = current_cost current_fitness = Linalg._fitness(matrix_a, matrix_b, next_W) current_cost = current_fitness + lambda_ * tracenorm if iter_counter > 0: # The first scores are messy cost_list = [L, L_bound, current_fitness, current_cost] costs.append(cost_list) while (current_fitness + epsilon_cost >= Linalg._intermediate_cost(matrix_a, matrix_b, next_W, Z, L)): if L > L_bound: print "Trace Norm Regression: numerical error detected at iteration "+str(iter_counter) break L = gamma * L try: next_W, tracenorm = Linalg._next_tracenorm_guess(matrix_a, matrix_b, lambda_, L, Z, at_times_a) except LinAlgError: print "LinAlgError caught in trace norm regression" linalg_error_caught = True break last_cost = current_cost current_fitness = Linalg._fitness(matrix_a, matrix_a, next_W) current_cost = current_fitness + lambda_*tracenorm if linalg_error_caught: break previous_W = W W = next_W previous_alpha = alpha alpha = (1.0 + sqrt(1.0 + 4.0*alpha*alpha))/2.0 Z = W # Z = W + ((alpha - 1)/alpha)*(W - previous_W) iter_counter += 1 sys.stdout.flush() W = np.real(W) return DenseMatrix(W), costs