def test_lasso_duality_gap(): A = np.eye(3) b = np.array([1.0, 2.0, 3.0]) regcoef = 2.0 # Checks at point x = [0, 0, 0] x = np.zeros(3) assert_almost_equal(0.77777777777777, oracles.lasso_duality_gap(x, A.dot(x) - b, A.T.dot(A.dot(x) - b), b, regcoef)) # Checks at point x = [1, 1, 1] x = np.ones(3) assert_almost_equal(3.0, oracles.lasso_duality_gap(x, A.dot(x) - b, A.T.dot(A.dot(x) - b), b, regcoef))
def barrier_method_lasso(A, b, reg_coef, x_0, u_0, tolerance=1e-5, tolerance_inner=1e-8, max_iter=100, max_iter_inner=20, t_0=1, gamma=10, c1=1e-4, lasso_duality_gap=None, trace=False, display=False): """ Log-barrier method for solving the problem: minimize f(x, u) := 1/2 * ||Ax - b||_2^2 + reg_coef * \sum_i u_i subject to -u_i <= x_i <= u_i. The method constructs the following barrier-approximation of the problem: phi_t(x, u) := t * f(x, u) - sum_i( log(u_i + x_i) + log(u_i - x_i) ) and minimize it as unconstrained problem by Newton's method. In the outer loop `t` is increased and we have a sequence of approximations { phi_t(x, u) } and solutions { (x_t, u_t)^{*} } which converges in `t` to the solution of the original problem. Parameters ---------- A : np.array Feature matrix for the regression problem. b : np.array Given vector of responses. reg_coef : float Regularization coefficient. x_0 : np.array Starting value for x in optimization algorithm. u_0 : np.array Starting value for u in optimization algorithm. tolerance : float Epsilon value for the outer loop stopping criterion: Stop the outer loop (which iterates over `k`) when `duality_gap(x_k) <= tolerance` tolerance_inner : float Epsilon value for the inner loop stopping criterion. Stop the inner loop (which iterates over `l`) when `|| \nabla phi_t(x_k^l) ||_2^2 <= tolerance_inner * \| \nabla \phi_t(x_k) \|_2^2 ` max_iter : int Maximum number of iterations for interior point method. max_iter_inner : int Maximum number of iterations for inner Newton's method. t_0 : float Starting value for `t`. gamma : float Multiplier for changing `t` during the iterations: t_{k + 1} = gamma * t_k. c1 : float Armijo's constant for line search in Newton's method. lasso_duality_gap : callable object or None. If calable the signature is lasso_duality_gap(x, Ax_b, ATAx_b, b, regcoef) Returns duality gap value for esimating the progress of method. trace : bool If True, the progress information is appended into history dictionary during training. Otherwise None is returned instead of history. display : bool If True, debug information is displayed during optimization. Printing format is up to a student and is not checked in any way. Returns ------- (x_star, u_star) : tuple of np.array The point found by the optimization procedure. message : string "success" or the description of error: - 'iterations_exceeded': if after max_iter iterations of the method x_k still doesn't satisfy the stopping criterion. - 'computational_error': in case of getting Infinity or None value during the computations. history : dictionary of lists or None Dictionary containing the progress information or None if trace=False. Dictionary has to be organized as follows: - history['time'] : list of floats, containing time in seconds passed from the start of the method - history['func'] : list of function values f(x_k) on every **outer** iteration of the algorithm - history['duality_gap'] : list of duality gaps - history['x'] : list of np.arrays, containing the trajectory of the algorithm. ONLY STORE IF x.size <= 2 """ def update_history(t, gap, z_k): history['time'].append(t) if x_0.size <= 2: history['x'].append(x_k) history['func'].append(oracle.func(z_k)) if lasso_duality_gap: history['duality_gap'].append(gap) history = defaultdict(list) if trace else None oracle = BarrierLassoOracle(A, b, reg_coef, t_0) x_k, u_k, t_k = x_0, u_0, t_0 n = A.shape[1] start = datetime.now() for k in range(max_iter): z_k = np.hstack([x_k, u_k]) if not (np.isfinite(z_k).all() and np.isfinite(oracle.func(z_k)) and np.isfinite(oracle.grad(z_k)).all()): return x_k, 'computational_error', history if lasso_duality_gap: Ax_b = A @ x_k - b ATAx_b = A.T @ Ax_b gap = lasso_duality_gap(x_k, Ax_b, ATAx_b, b, reg_coef) if gap <= tolerance: break else: gap = None if display: print(x_k) if trace: t = (datetime.now() - start).total_seconds() z_k = np.hstack([x_k, u_k]) update_history(t, gap, z_k) if gap <= tolerance: return (x_k, u_k), 'success', history z_k = np.hstack([x_k, u_k]) z_k, message, hist = newton(oracle, z_k, tolerance_inner, max_iter_inner, line_search_options={'c1': c1}, trace=True) x_k, u_k = z_k[:n], z_k[n:] t_k = min(n / tolerance + 1, t_k * gamma) oracle.update_tau(t_k) z_k = np.hstack([x_k, u_k]) if not (np.isfinite(z_k).all() and np.isfinite(oracle.func(z_k)) and np.isfinite(oracle.grad(z_k)).all()): return x_k, 'computational_error', history if lasso_duality_gap: Ax_b = A @ x_k - b ATAx_b = A.T @ Ax_b gap = lasso_duality_gap(x_k, Ax_b, ATAx_b, b, reg_coef) if trace: t = (datetime.now() - start).total_seconds() z_k = np.hstack([x_k, u_k]) update_history(t, gap, z_k) if display: print(x_k) if gap <= tolerance: return (x_k, u_k), 'success', history return (x_k, u_k), 'iterations_exceeded', history