def test_callback(): """Make sure that the algorithm exists when the callback returns False.""" def cb(_): return False l1ball = cp.constraint.L1Ball(1) f = cp.loss.SquareLoss(A, b) opt = cp.minimize_frank_wolfe(f.f_grad, np.zeros(n_features), l1ball.lmo, callback=cb) assert opt.nit < 2
def test_fw_api(): """Check that FW takes the right arguments and raises the right exceptions.""" # test that the algorithm does not fail if x0 # is a tuple f = copt.loss.LogLoss(A, b, 1.0 / n_samples) cb = cp.utils.Trace(f) alpha = 1.0 l1ball = copt.constraint.L1Ball(alpha) cp.minimize_frank_wolfe( f.f_grad, [0] * n_features, l1ball.lmo, tol=0, lipschitz=f.lipschitz, callback=cb, ) # check that we riase an exception when the DR step-size is used # but no lipschitz constant is given with pytest.raises(ValueError): cp.minimize_frank_wolfe(f.f_grad, [0] * n_features, l1ball.lmo, step="DR")
def test_pairwise_fw(obj, step, alpha): """Test the Pairwise FW method.""" f = obj(A, b, 1.0 / n_samples) l1ball = copt.constraint.L1Ball(alpha) x0 = np.zeros(A.shape[1]) x0[0] = alpha cb = cp.utils.Trace(f) opt = cp.minimize_frank_wolfe( f.f_grad, x0, l1ball.lmo_pairwise, step=step, lipschitz=f.lipschitz, callback=cb ) assert np.isfinite(opt.x).sum() == n_features ss = 1 / f.lipschitz grad = f.f_grad(opt.x)[1] grad_map = (opt.x - l1ball.prox(opt.x - ss * grad, ss)) / ss assert np.linalg.norm(grad_map) < 0.2
def test_fw_backtrack(obj, step, alpha): """Test FW with different options of the line-search strategy.""" f = obj(A, b, 1.0 / n_samples) traceball = copt.constraint.TraceBall(alpha, (4, 4)) opt = cp.minimize_frank_wolfe( f.f_grad, np.zeros(n_features), traceball.lmo, tol=0, lipschitz=f.lipschitz, step=step, max_iter=1000, ) assert np.isfinite(opt.x).sum() == n_features ss = 1 / f.lipschitz grad = f.f_grad(opt.x)[1] grad_map = (opt.x - traceball.prox(opt.x - ss * grad, ss)) / ss assert np.linalg.norm(grad_map) < 0.4
def test_fw_l1(loss_grad, alpha): """Test result of FW algorithm with L1 constraint.""" f = loss_grad(A, b, 1.0 / n_samples) cb = cp.utils.Trace(f) l1ball = copt.constraint.L1Ball(alpha) opt = cp.minimize_frank_wolfe( f.f_grad, np.zeros(n_features), l1ball.lmo, tol=1e-3, lipschitz=f.lipschitz, callback=cb, ) assert np.isfinite(opt.x).sum() == n_features ss = 1 / f.lipschitz grad = f.f_grad(opt.x)[1] grad_map = (opt.x - l1ball.prox(opt.x - ss * grad, ss)) / ss assert np.linalg.norm(grad_map) < 0.3
def test_fw_backtrack(obj, bt): """Test FW with different options of the line-search strategy.""" f = obj(A, b, 1. / n_samples) alpha = 1. traceball = cp.utils.TraceBall(alpha, (4, 4)) opt = cp.minimize_frank_wolfe( f.f_grad, traceball.lmo, np.zeros(n_features), tol=0, # max_iter=5000, lipschitz=f.lipschitz, line_search=bt) assert np.isfinite(opt.x).sum() == n_features ss = 1 / f.lipschitz grad = f.f_grad(opt.x)[1] grad_map = (opt.x - traceball.prox(opt.x - ss * grad, ss)) / ss assert np.linalg.norm(grad_map) < 1e-2
prev_overlap = overlap[-1] if np.linalg.norm(dt_prev[0] - s_t) == 0: overlap.append(prev_overlap + 1) else: overlap.append(prev_overlap) dt_prev[0] = s_t else: overlap.append(0) dt_prev.append(s_t) if label.startswith("Frank-Wolfe"): cp.minimize_frank_wolfe( f.f_grad, x0, l1_ball.lmo, callback=trace, max_iter=50, step_size=step_size, verbose=True, lipschitz=f.lipschitz, ) elif label.startswith("Pairwise"): pass ax.plot(overlap, label=label, marker=marker, markevery=7 + i) ax.legend() ax.set_xlabel("number of iterations") ax.set_ylabel("LMO overlap") ax.set_title(dataset_title) fig.tight_layout() # otherwise the right y-label is slightly clipped ax.grid() # plt.legend() plt.show()
n_samples, n_features = X.shape l1_ball = copt.constraint.L1Ball(d["alpha"]) f = copt.loss.LogLoss(X, y) x0 = np.zeros(n_features) x0[0] = d["alpha"] # start from a (random) vertex for step, label, marker in variants_fw: cb = cp.utils.Trace(f) sol = cp.minimize_frank_wolfe( f.f_grad, x0, l1_ball.lmo_pairwise, callback=cb, step=step, lipschitz=f.lipschitz, max_iter=d["max_iter"], verbose=True, tol=0, ) plt.plot( cb.trace_time, np.array(cb.trace_fx) - d["f_star"], label=label, marker=marker, markevery=10, ) print("Sparsity of solution: %s" % np.mean(np.abs(sol.x) > 1e-8))
X, y = load_data() n_samples, n_features = X.shape l1_ball = cp.utils.L1Ball(alpha) f = cp.utils.LogLoss(X, y) x0 = np.zeros(n_features) for step_size, label, marker in variants_fw: cb = cp.utils.Trace(f) sol = cp.minimize_frank_wolfe( f.f_grad, x0, l1_ball.lmo, callback=cb, step_size=step_size, lipschitz=f.lipschitz, # max_iter=1000 ) plt.plot(cb.trace_time, cb.trace_fx, label=label, marker=marker, markevery=10) print("Sparsity of solution: %s" % np.mean(np.abs(sol.x) > 1e-8)) plt.legend() plt.xlabel("Time (in seconds)") plt.ylabel("Objective function")
# sparse.linalg.norm to make the comparison prev_overlap = overlap[-1] if np.linalg.norm(st_prev[0] - s_t) == 0: overlap.append(prev_overlap + 1) else: overlap.append(prev_overlap) st_prev[0] = s_t else: overlap.append(0) st_prev.append(s_t) cp.minimize_frank_wolfe( f.f_grad, x0, l1_ball.lmo, callback=trace, max_iter=int(1e4), step=step, verbose=True, lipschitz=f.lipschitz, ) ax.plot(overlap, label=label) ax.yaxis.set_major_locator(MaxNLocator(integer=True)) ax.legend() ax.set_xlabel("number of iterations") ax.set_ylabel("LMO overlap") ax.set_title(dataset_title) fig.tight_layout() # otherwise the right y-label is slightly clipped ax.grid() # plt.legend() plt.show()
plt.figure() print("Running on the %s dataset" % dataset_title) X, y = load_data() n_samples, n_features = X.shape l1_ball = copt.constraint.L1Ball(alpha) f = copt.loss.LogLoss(X, y) x0 = np.zeros(n_features) for step, label in variants_fw: cb = cp.utils.Trace(f) sol = cp.minimize_frank_wolfe(f.f_grad, x0, l1_ball.lmo, callback=cb, step=step, lipschitz=f.lipschitz) plt.plot(cb.trace_time, cb.trace_fx, label=label, markevery=10) print("Sparsity of solution: %s" % np.mean(np.abs(sol.x) > 1e-8)) plt.legend() plt.xlabel("Time (in seconds)") plt.ylabel("Objective function") plt.title(dataset_title) plt.tight_layout() # otherwise the right y-label is slightly clipped plt.xlim((0, 0.7 * cb.trace_time[-1])) # for aesthetics plt.grid() plt.show()
trace_step_size = [] trace_curavature = [] def cb(kw): trace_step_size.append(kw['step_size']) hessian = splinalg.LinearOperator(shape=(n_features, n_features), matvec=f.Hessian(kw['x'])) s, _ = splinalg.eigsh(hessian, k=1) trace_curavature.append(s) out = cp.minimize_frank_wolfe(f.f_grad, l1_ball.lmo, x0, callback=cb, max_iter=1000, line_search=exact_ls) # Focus on the last 4/5, since the first iterations # tend to have a disproportionally large step-size n = len(trace_step_size) // 5 trace_step_size = trace_step_size[n:] trace_curavature = trace_curavature[n:] fig, ax1 = plt.subplots() color = '#67a9cf' ax1.set_xlabel('number of iterations') ax1.set_ylabel('step-size', color=color) ax1.plot(n + np.arange(len(trace_step_size)),