def test_lasso_duality_gap():
A = np.eye(3)
b = np.array([1.0, 2.0, 3.0])
regcoef = 2.0
# Checks at point x = [0, 0, 0]
x = np.zeros(3)
assert_almost_equal(0.77777777777777,
oracles.lasso_duality_gap(x, A.dot(x) - b,
A.T.dot(A.dot(x) - b),
b, regcoef))
# Checks at point x = [1, 1, 1]
x = np.ones(3)
assert_almost_equal(3.0, oracles.lasso_duality_gap(x, A.dot(x) - b,
A.T.dot(A.dot(x) - b),
b, regcoef))
def barrier_method_lasso(A,
b,
reg_coef,
x_0,
u_0,
tolerance=1e-5,
tolerance_inner=1e-8,
max_iter=100,
max_iter_inner=20,
t_0=1,
gamma=10,
c1=1e-4,
lasso_duality_gap=None,
trace=False,
display=False):
"""
Log-barrier method for solving the problem:
minimize f(x, u) := 1/2 * ||Ax - b||_2^2 + reg_coef * \sum_i u_i
subject to -u_i <= x_i <= u_i.
The method constructs the following barrier-approximation of the problem:
phi_t(x, u) := t * f(x, u) - sum_i( log(u_i + x_i) + log(u_i - x_i) )
and minimize it as unconstrained problem by Newton's method.
In the outer loop `t` is increased and we have a sequence of approximations
{ phi_t(x, u) } and solutions { (x_t, u_t)^{*} } which converges in `t`
to the solution of the original problem.
Parameters
----------
A : np.array
Feature matrix for the regression problem.
b : np.array
Given vector of responses.
reg_coef : float
Regularization coefficient.
x_0 : np.array
Starting value for x in optimization algorithm.
u_0 : np.array
Starting value for u in optimization algorithm.
tolerance : float
Epsilon value for the outer loop stopping criterion:
Stop the outer loop (which iterates over `k`) when
`duality_gap(x_k) <= tolerance`
tolerance_inner : float
Epsilon value for the inner loop stopping criterion.
Stop the inner loop (which iterates over `l`) when
`|| \nabla phi_t(x_k^l) ||_2^2 <= tolerance_inner * \| \nabla \phi_t(x_k) \|_2^2 `
max_iter : int
Maximum number of iterations for interior point method.
max_iter_inner : int
Maximum number of iterations for inner Newton's method.
t_0 : float
Starting value for `t`.
gamma : float
Multiplier for changing `t` during the iterations:
t_{k + 1} = gamma * t_k.
c1 : float
Armijo's constant for line search in Newton's method.
lasso_duality_gap : callable object or None.
If calable the signature is lasso_duality_gap(x, Ax_b, ATAx_b, b, regcoef)
Returns duality gap value for esimating the progress of method.
trace : bool
If True, the progress information is appended into history dictionary
during training. Otherwise None is returned instead of history.
display : bool
If True, debug information is displayed during optimization.
Printing format is up to a student and is not checked in any way.
Returns
-------
(x_star, u_star) : tuple of np.array
The point found by the optimization procedure.
message : string
"success" or the description of error:
- 'iterations_exceeded': if after max_iter iterations of the method x_k still doesn't satisfy
the stopping criterion.
- 'computational_error': in case of getting Infinity or None value during the computations.
history : dictionary of lists or None
Dictionary containing the progress information or None if trace=False.
Dictionary has to be organized as follows:
- history['time'] : list of floats, containing time in seconds passed from the start of the method
- history['func'] : list of function values f(x_k) on every **outer** iteration of the algorithm
- history['duality_gap'] : list of duality gaps
- history['x'] : list of np.arrays, containing the trajectory of the algorithm. ONLY STORE IF x.size <= 2
"""
def update_history(t, gap, z_k):
history['time'].append(t)
if x_0.size <= 2:
history['x'].append(x_k)
history['func'].append(oracle.func(z_k))
if lasso_duality_gap:
history['duality_gap'].append(gap)
history = defaultdict(list) if trace else None
oracle = BarrierLassoOracle(A, b, reg_coef, t_0)
x_k, u_k, t_k = x_0, u_0, t_0
n = A.shape[1]
start = datetime.now()
for k in range(max_iter):
z_k = np.hstack([x_k, u_k])
if not (np.isfinite(z_k).all() and np.isfinite(oracle.func(z_k))
and np.isfinite(oracle.grad(z_k)).all()):
return x_k, 'computational_error', history
if lasso_duality_gap:
Ax_b = A @ x_k - b
ATAx_b = A.T @ Ax_b
gap = lasso_duality_gap(x_k, Ax_b, ATAx_b, b, reg_coef)
if gap <= tolerance:
break
else:
gap = None
if display:
print(x_k)
if trace:
t = (datetime.now() - start).total_seconds()
z_k = np.hstack([x_k, u_k])
update_history(t, gap, z_k)
if gap <= tolerance:
return (x_k, u_k), 'success', history
z_k = np.hstack([x_k, u_k])
z_k, message, hist = newton(oracle,
z_k,
tolerance_inner,
max_iter_inner,
line_search_options={'c1': c1},
trace=True)
x_k, u_k = z_k[:n], z_k[n:]
t_k = min(n / tolerance + 1, t_k * gamma)
oracle.update_tau(t_k)
z_k = np.hstack([x_k, u_k])
if not (np.isfinite(z_k).all() and np.isfinite(oracle.func(z_k))
and np.isfinite(oracle.grad(z_k)).all()):
return x_k, 'computational_error', history
if lasso_duality_gap:
Ax_b = A @ x_k - b
ATAx_b = A.T @ Ax_b
gap = lasso_duality_gap(x_k, Ax_b, ATAx_b, b, reg_coef)
if trace:
t = (datetime.now() - start).total_seconds()
z_k = np.hstack([x_k, u_k])
update_history(t, gap, z_k)
if display:
print(x_k)
if gap <= tolerance:
return (x_k, u_k), 'success', history
return (x_k, u_k), 'iterations_exceeded', history
