def test_callback():
"""Make sure that the algorithm exists when the callback returns False."""
def cb(_):
return False
l1ball = cp.constraint.L1Ball(1)
f = cp.loss.SquareLoss(A, b)
opt = cp.minimize_frank_wolfe(f.f_grad,
np.zeros(n_features),
l1ball.lmo,
callback=cb)
assert opt.nit < 2
def test_fw_api():
"""Check that FW takes the right arguments and raises the right exceptions."""
# test that the algorithm does not fail if x0
# is a tuple
f = copt.loss.LogLoss(A, b, 1.0 / n_samples)
cb = cp.utils.Trace(f)
alpha = 1.0
l1ball = copt.constraint.L1Ball(alpha)
cp.minimize_frank_wolfe(
f.f_grad,
[0] * n_features,
l1ball.lmo,
tol=0,
lipschitz=f.lipschitz,
callback=cb,
)
# check that we riase an exception when the DR step-size is used
# but no lipschitz constant is given
with pytest.raises(ValueError):
cp.minimize_frank_wolfe(f.f_grad, [0] * n_features, l1ball.lmo, step="DR")
def test_pairwise_fw(obj, step, alpha):
"""Test the Pairwise FW method."""
f = obj(A, b, 1.0 / n_samples)
l1ball = copt.constraint.L1Ball(alpha)
x0 = np.zeros(A.shape[1])
x0[0] = alpha
cb = cp.utils.Trace(f)
opt = cp.minimize_frank_wolfe(
f.f_grad, x0, l1ball.lmo_pairwise, step=step, lipschitz=f.lipschitz, callback=cb
)
assert np.isfinite(opt.x).sum() == n_features
ss = 1 / f.lipschitz
grad = f.f_grad(opt.x)[1]
grad_map = (opt.x - l1ball.prox(opt.x - ss * grad, ss)) / ss
assert np.linalg.norm(grad_map) < 0.2
def test_fw_backtrack(obj, step, alpha):
"""Test FW with different options of the line-search strategy."""
f = obj(A, b, 1.0 / n_samples)
traceball = copt.constraint.TraceBall(alpha, (4, 4))
opt = cp.minimize_frank_wolfe(
f.f_grad,
np.zeros(n_features),
traceball.lmo,
tol=0,
lipschitz=f.lipschitz,
step=step,
max_iter=1000,
)
assert np.isfinite(opt.x).sum() == n_features
ss = 1 / f.lipschitz
grad = f.f_grad(opt.x)[1]
grad_map = (opt.x - traceball.prox(opt.x - ss * grad, ss)) / ss
assert np.linalg.norm(grad_map) < 0.4
def test_fw_l1(loss_grad, alpha):
"""Test result of FW algorithm with L1 constraint."""
f = loss_grad(A, b, 1.0 / n_samples)
cb = cp.utils.Trace(f)
l1ball = copt.constraint.L1Ball(alpha)
opt = cp.minimize_frank_wolfe(
f.f_grad,
np.zeros(n_features),
l1ball.lmo,
tol=1e-3,
lipschitz=f.lipschitz,
callback=cb,
)
assert np.isfinite(opt.x).sum() == n_features
ss = 1 / f.lipschitz
grad = f.f_grad(opt.x)[1]
grad_map = (opt.x - l1ball.prox(opt.x - ss * grad, ss)) / ss
assert np.linalg.norm(grad_map) < 0.3
def test_fw_backtrack(obj, bt):
"""Test FW with different options of the line-search strategy."""
f = obj(A, b, 1. / n_samples)
alpha = 1.
traceball = cp.utils.TraceBall(alpha, (4, 4))
opt = cp.minimize_frank_wolfe(
f.f_grad,
traceball.lmo,
np.zeros(n_features),
tol=0,
# max_iter=5000,
lipschitz=f.lipschitz,
line_search=bt)
assert np.isfinite(opt.x).sum() == n_features
ss = 1 / f.lipschitz
grad = f.f_grad(opt.x)[1]
grad_map = (opt.x - traceball.prox(opt.x - ss * grad, ss)) / ss
assert np.linalg.norm(grad_map) < 1e-2
prev_overlap = overlap[-1]
if np.linalg.norm(dt_prev[0] - s_t) == 0:
overlap.append(prev_overlap + 1)
else:
overlap.append(prev_overlap)
dt_prev[0] = s_t
else:
overlap.append(0)
dt_prev.append(s_t)
if label.startswith("Frank-Wolfe"):
cp.minimize_frank_wolfe(
f.f_grad,
x0,
l1_ball.lmo,
callback=trace,
max_iter=50,
step_size=step_size,
verbose=True,
lipschitz=f.lipschitz,
)
elif label.startswith("Pairwise"):
pass
ax.plot(overlap, label=label, marker=marker, markevery=7 + i)
ax.legend()
ax.set_xlabel("number of iterations")
ax.set_ylabel("LMO overlap")
ax.set_title(dataset_title)
fig.tight_layout() # otherwise the right y-label is slightly clipped
ax.grid()
# plt.legend()
plt.show()
n_samples, n_features = X.shape
l1_ball = copt.constraint.L1Ball(d["alpha"])
f = copt.loss.LogLoss(X, y)
x0 = np.zeros(n_features)
x0[0] = d["alpha"] # start from a (random) vertex
for step, label, marker in variants_fw:
cb = cp.utils.Trace(f)
sol = cp.minimize_frank_wolfe(
f.f_grad,
x0,
l1_ball.lmo_pairwise,
callback=cb,
step=step,
lipschitz=f.lipschitz,
max_iter=d["max_iter"],
verbose=True,
tol=0,
)
plt.plot(
cb.trace_time,
np.array(cb.trace_fx) - d["f_star"],
label=label,
marker=marker,
markevery=10,
)
print("Sparsity of solution: %s" % np.mean(np.abs(sol.x) > 1e-8))
X, y = load_data()
n_samples, n_features = X.shape
l1_ball = cp.utils.L1Ball(alpha)
f = cp.utils.LogLoss(X, y)
x0 = np.zeros(n_features)
for step_size, label, marker in variants_fw:
cb = cp.utils.Trace(f)
sol = cp.minimize_frank_wolfe(
f.f_grad,
x0,
l1_ball.lmo,
callback=cb,
step_size=step_size,
lipschitz=f.lipschitz,
# max_iter=1000
)
plt.plot(cb.trace_time,
cb.trace_fx,
label=label,
marker=marker,
markevery=10)
print("Sparsity of solution: %s" % np.mean(np.abs(sol.x) > 1e-8))
plt.legend()
plt.xlabel("Time (in seconds)")
plt.ylabel("Objective function")
# sparse.linalg.norm to make the comparison
prev_overlap = overlap[-1]
if np.linalg.norm(st_prev[0] - s_t) == 0:
overlap.append(prev_overlap + 1)
else:
overlap.append(prev_overlap)
st_prev[0] = s_t
else:
overlap.append(0)
st_prev.append(s_t)
cp.minimize_frank_wolfe(
f.f_grad,
x0,
l1_ball.lmo,
callback=trace,
max_iter=int(1e4),
step=step,
verbose=True,
lipschitz=f.lipschitz,
)
ax.plot(overlap, label=label)
ax.yaxis.set_major_locator(MaxNLocator(integer=True))
ax.legend()
ax.set_xlabel("number of iterations")
ax.set_ylabel("LMO overlap")
ax.set_title(dataset_title)
fig.tight_layout() # otherwise the right y-label is slightly clipped
ax.grid()
# plt.legend()
plt.show()
plt.figure()
print("Running on the %s dataset" % dataset_title)
X, y = load_data()
n_samples, n_features = X.shape
l1_ball = copt.constraint.L1Ball(alpha)
f = copt.loss.LogLoss(X, y)
x0 = np.zeros(n_features)
for step, label in variants_fw:
cb = cp.utils.Trace(f)
sol = cp.minimize_frank_wolfe(f.f_grad,
x0,
l1_ball.lmo,
callback=cb,
step=step,
lipschitz=f.lipschitz)
plt.plot(cb.trace_time, cb.trace_fx, label=label, markevery=10)
print("Sparsity of solution: %s" % np.mean(np.abs(sol.x) > 1e-8))
plt.legend()
plt.xlabel("Time (in seconds)")
plt.ylabel("Objective function")
plt.title(dataset_title)
plt.tight_layout() # otherwise the right y-label is slightly clipped
plt.xlim((0, 0.7 * cb.trace_time[-1])) # for aesthetics
plt.grid()
plt.show()
trace_step_size = []
trace_curavature = []
def cb(kw):
trace_step_size.append(kw['step_size'])
hessian = splinalg.LinearOperator(shape=(n_features, n_features),
matvec=f.Hessian(kw['x']))
s, _ = splinalg.eigsh(hessian, k=1)
trace_curavature.append(s)
out = cp.minimize_frank_wolfe(f.f_grad,
l1_ball.lmo,
x0,
callback=cb,
max_iter=1000,
line_search=exact_ls)
# Focus on the last 4/5, since the first iterations
# tend to have a disproportionally large step-size
n = len(trace_step_size) // 5
trace_step_size = trace_step_size[n:]
trace_curavature = trace_curavature[n:]
fig, ax1 = plt.subplots()
color = '#67a9cf'
ax1.set_xlabel('number of iterations')
ax1.set_ylabel('step-size', color=color)
ax1.plot(n + np.arange(len(trace_step_size)),
