def test_forward_backward(self):
"""
Test forward-backward splitting algorithm without acceleration, and
with L1-norm, L2-norm, and dummy functions.
"""
y = [4., 5., 6., 7.]
solver = solvers.forward_backward(accel=acceleration.dummy())
param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 35)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 4)
# Sanity check
f3 = functions.dummy()
x0 = np.zeros((4,))
self.assertRaises(ValueError, solver.pre, [f1, f2, f3], x0)
def test_forward_backward(self):
"""
Test forward-backward splitting algorithm without acceleration, and
with L1-norm, L2-norm, and dummy functions.
"""
y = [4., 5., 6., 7.]
solver = solvers.forward_backward(accel=acceleration.dummy())
param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 35)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 4)
# Sanity check
f3 = functions.dummy()
x0 = np.zeros((4, ))
self.assertRaises(ValueError, solver.pre, [f1, f2, f3], x0)
def test_solver(self):
"""
Base solver class.
"""
funs = [functions.dummy(), functions.dummy()]
x0 = np.zeros((4,))
s = solvers.solver()
s.sol = x0
self.assertRaises(ValueError, s.__init__, -1.)
self.assertRaises(NotImplementedError, s.pre, funs, x0)
self.assertRaises(NotImplementedError, s._algo)
self.assertRaises(NotImplementedError, s.post)
def test_solver(self):
"""
Base solver class.
"""
funs = [functions.dummy(), functions.dummy()]
x0 = np.zeros((4, ))
s = solvers.solver()
s.sol = x0
self.assertRaises(ValueError, s.__init__, -1.)
self.assertRaises(NotImplementedError, s.pre, funs, x0)
self.assertRaises(NotImplementedError, s._algo)
self.assertRaises(NotImplementedError, s.post)
def test_douglas_rachford(self):
"""
Test douglas-rachford solver with L1-norm, L2-norm and dummy functions.
"""
y = [4, 5, 6, 7]
solver = solvers.douglas_rachford()
param = {'solver': solver, 'verbosity': 'NONE'}
# L2-norm prox and dummy prox.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 35)
# L2-norm prox and L1-norm prox.
f1 = functions.norm_l2(y=y)
f2 = functions.norm_l1(y=y)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 4)
# Sanity checks
x0 = np.zeros((4,))
solver.lambda_ = 2.
self.assertRaises(ValueError, solver.pre, [f1, f2], x0)
solver.lambda_ = -2.
self.assertRaises(ValueError, solver.pre, [f1, f2], x0)
self.assertRaises(ValueError, solver.pre, [f1, f2, f1], x0)
def test_douglas_rachford(self):
"""
Test douglas-rachford solver with L1-norm, L2-norm and dummy functions.
"""
y = [4, 5, 6, 7]
solver = solvers.douglas_rachford()
param = {'solver': solver, 'verbosity': 'NONE'}
# L2-norm prox and dummy prox.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 35)
# L2-norm prox and L1-norm prox.
f1 = functions.norm_l2(y=y)
f2 = functions.norm_l1(y=y)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 4)
# Sanity checks
x0 = np.zeros((4, ))
solver.lambda_ = 2.
self.assertRaises(ValueError, solver.pre, [f1, f2], x0)
solver.lambda_ = -2.
self.assertRaises(ValueError, solver.pre, [f1, f2], x0)
self.assertRaises(ValueError, solver.pre, [f1, f2, f1], x0)
def test_mlfbf(self):
"""
Test the MLFBF solver with arbitrarily selected functions.
"""
x = [1., 1., 1.]
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
max_step = 1 / (1 + np.linalg.norm(L, 2))
solver = solvers.mlfbf(L=L, step=max_step / 2.)
params = {'solver': solver, 'verbosity': 'NONE'}
def x0():
return np.zeros(len(x))
# L2-norm prox and dummy prox.
f = functions.dummy()
f._prox = lambda x, T: np.maximum(np.zeros(len(x)), x)
g = functions.norm_l2(lambda_=0.5)
h = functions.norm_l2(y=np.array([294, 390, 361]), lambda_=0.5)
ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Same test, but with callable L
solver = solvers.mlfbf(L=lambda x: np.dot(L, x),
Lt=lambda y: np.dot(L.T, y),
d0=np.dot(L, x0()),
step=max_step / 2.)
ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Sanity check
self.assertRaises(ValueError, solver.pre, [f, g], x0())
def test_mlfbf(self):
"""
Test the MLFBF solver with arbitrarily selected functions.
"""
x = [1., 1., 1.]
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
max_step = 1 / (1 + np.linalg.norm(L, 2))
solver = solvers.mlfbf(L=L, step=max_step / 2.)
params = {'solver': solver, 'verbosity': 'NONE'}
def x0(): return np.zeros(len(x))
# L2-norm prox and dummy prox.
f = functions.dummy()
f._prox = lambda x, T: np.maximum(np.zeros(len(x)), x)
g = functions.norm_l2(lambda_=0.5)
h = functions.norm_l2(y=np.array([294, 390, 361]), lambda_=0.5)
ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Same test, but with callable L
solver = solvers.mlfbf(L=lambda x: np.dot(L, x),
Lt=lambda y: np.dot(L.T, y),
d0=np.dot(L, x0()),
step=max_step / 2.)
ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Sanity check
self.assertRaises(ValueError, solver.pre, [f, g], x0())
def test_regularized_nonlinear(self):
"""
Test gradient descent solver with regularized non-linear acceleration,
solving problems with L2-norm functions.
"""
dim = 25
np.random.seed(0)
x0 = np.random.rand(dim)
xstar = np.random.rand(dim)
x0 = xstar + 5. * (x0 - xstar) / np.linalg.norm(x0 - xstar)
A = np.random.rand(dim, dim)
step = 1 / np.linalg.norm(np.dot(A.T, A))
accel = acceleration.regularized_nonlinear(k=5)
solver = solvers.gradient_descent(step=step, accel=accel)
param = {'solver': solver, 'rtol': 0,
'maxit': 200, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(lambda_=0.5, A=A, y=np.dot(A, xstar))
f2 = functions.dummy()
ret = solvers.solve([f1, f2], x0, **param)
pctdiff = 100 * np.sum((xstar - ret['sol'])**2) / np.sum(xstar**2)
nptest.assert_array_less(pctdiff, 1.91)
# Sanity checks
accel = acceleration.regularized_nonlinear()
self.assertRaises(ValueError, accel.__init__, 10, ['not', 'good'])
self.assertRaises(ValueError, accel.__init__, 10, 'nope')
def test_accel(self):
"""
Test base acceleration scheme class
"""
funs = [functions.dummy(), functions.dummy()]
x0 = np.zeros((4, ))
a = acceleration.accel()
s = solvers.forward_backward()
o = [[1., 2.], [0., 1.]]
n = 2
self.assertRaises(NotImplementedError, a.pre, funs, x0)
self.assertRaises(NotImplementedError, a.update_step, s, o, n)
self.assertRaises(NotImplementedError, a.update_sol, s, o, n)
self.assertRaises(NotImplementedError, a.post)
def test_backtracking(self):
"""
Test forward-backward splitting solver with backtracking, solving
problems with L1-norm, L2-norm, and dummy functions.
"""
# Test constructor sanity
a = acceleration.backtracking()
self.assertRaises(ValueError, a.__init__, 2.)
self.assertRaises(ValueError, a.__init__, -2.)
y = [4., 5., 6., 7.]
accel = acceleration.backtracking()
step = 10 # Make sure backtracking is called
solver = solvers.forward_backward(accel=accel, step=step)
param = {'solver': solver, 'atol': 1e-32, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'ATOL')
self.assertEqual(ret['niter'], 13)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'ATOL')
self.assertEqual(ret['niter'], 4)
def test_forward_backward_fista_backtracking(self):
"""
Test forward-backward splitting solver with fista acceleration and
backtracking, solving problems with L1-norm, L2-norm, and dummy
functions.
"""
y = [4., 5., 6., 7.]
accel = acceleration.fista_backtracking()
solver = solvers.forward_backward(accel=accel)
param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 60)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 3)
def test_backtracking(self):
"""
Test forward-backward splitting solver with backtracking, solving
problems with L1-norm, L2-norm, and dummy functions.
"""
# Test constructor sanity
a = acceleration.backtracking()
self.assertRaises(ValueError, a.__init__, 2.)
self.assertRaises(ValueError, a.__init__, -2.)
y = [4., 5., 6., 7.]
accel = acceleration.backtracking()
step = 10 # Make sure backtracking is called
solver = solvers.forward_backward(accel=accel, step=step)
param = {'solver': solver, 'atol': 1e-32, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'ATOL')
self.assertEqual(ret['niter'], 13)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'ATOL')
self.assertLessEqual(ret['niter'], 4) # win64 takes one iteration
def test_regularized_nonlinear(self):
"""
Test gradient descent solver with regularized non-linear acceleration,
solving problems with L2-norm functions.
"""
dim = 25
np.random.seed(0)
x0 = np.random.rand(dim)
xstar = np.random.rand(dim)
x0 = xstar + 5. * (x0 - xstar) / np.linalg.norm(x0 - xstar)
A = np.random.rand(dim, dim)
step = 1 / np.linalg.norm(np.dot(A.T, A))
accel = acceleration.regularized_nonlinear(k=5)
solver = solvers.gradient_descent(step=step, accel=accel)
param = {
'solver': solver,
'rtol': 0,
'maxit': 200,
'verbosity': 'NONE'
}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(lambda_=0.5, A=A, y=np.dot(A, xstar))
f2 = functions.dummy()
ret = solvers.solve([f1, f2], x0, **param)
pctdiff = 100 * np.sum((xstar - ret['sol'])**2) / np.sum(xstar**2)
nptest.assert_array_less(pctdiff, 1.91)
# Sanity checks
accel = acceleration.regularized_nonlinear()
self.assertRaises(ValueError, accel.__init__, 10, ['not', 'good'])
self.assertRaises(ValueError, accel.__init__, 10, 'nope')
def test_forward_backward_fista_backtracking(self):
"""
Test forward-backward splitting solver with fista acceleration and
backtracking, solving problems with L1-norm, L2-norm, and dummy
functions.
"""
y = [4., 5., 6., 7.]
accel = acceleration.fista_backtracking()
solver = solvers.forward_backward(accel=accel)
param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 60)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 3)
def test_accel(self):
"""
Test base acceleration scheme class
"""
funs = [functions.dummy(), functions.dummy()]
x0 = np.zeros((4,))
a = acceleration.accel()
s = solvers.forward_backward()
o = [[1., 2.], [0., 1.]]
n = 2
self.assertRaises(NotImplementedError, a.pre, funs, x0)
self.assertRaises(NotImplementedError, a.update_step, s, o, n)
self.assertRaises(NotImplementedError, a.update_sol, s, o, n)
self.assertRaises(NotImplementedError, a.post)
def test_primal_dual_solver_comparison(self):
"""
Test that all primal-dual solvers return the same and correct solution.
I had to create this separate function because the primal-dual solvers
were too slow for the problem above.
"""
# Convex functions.
y = np.random.randn(3)
L = np.random.randn(4, 3)
sol = y
y2 = L.dot(y)
f1 = functions.norm_l1(y=y)
f2 = functions.norm_l2(y=y2)
f3 = functions.dummy()
# Solvers.
step = 0.5 / (1 + np.linalg.norm(L, 2))
slvs = []
slvs.append(solvers.mlfbf(step=step, L=L))
slvs.append(solvers.projection_based(step=step, L=L))
# Compare solutions.
niter = 1000
params = {'rtol': 0, 'verbosity': 'NONE', 'maxit': niter}
for solver in slvs:
x0 = np.zeros(len(y))
if type(solver) is solvers.mlfbf:
ret = solvers.solve([f1, f2, f3], x0, solver, **params)
else:
ret = solvers.solve([f1, f2], x0, solver, **params)
nptest.assert_allclose(ret['sol'], sol)
self.assertEqual(ret['niter'], niter)
# The initial value was not modified.
nptest.assert_array_equal(x0, np.zeros(len(y)))
if type(solver) is solvers.mlfbf:
ret = solvers.solve([f1, f2, f3],
x0,
solver,
inplace=True,
**params)
else:
ret = solvers.solve([f1, f2],
x0,
solver,
inplace=True,
**params)
# The initial value was modified.
self.assertIs(ret['sol'], x0)
nptest.assert_allclose(ret['sol'], sol)
def test_dummy(self):
"""
Test the dummy derived class.
All the methods should return 0.
"""
f = functions.dummy()
self.assertEqual(f.eval(34), 0)
nptest.assert_array_equal(f.grad(34), [0])
nptest.assert_array_equal(f.prox(34, 1), [34])
x = [34, 2, 1.0, -10.2]
self.assertEqual(f.eval(x), 0)
nptest.assert_array_equal(f.grad(x), np.zeros(len(x)))
nptest.assert_array_equal(f.prox(x, 1), x)
def test_prox_star(self):
n = 10
x = 3 * np.random.randn(n, 1)
f = functions.norm_l1()
f2 = functions.dummy()
f2.prox = lambda x, T: functions._prox_star(f, x, T)
gamma = np.random.rand()
p1 = f.prox(x, gamma)
p2 = functions._prox_star(f2, x, gamma)
np.testing.assert_array_almost_equal(p1, p2)
p1 = f.prox(x, gamma) - x
p2 = -gamma * f2.prox(x / gamma, 1 / gamma)
np.testing.assert_array_almost_equal(p1, p2)
def inner_solve(self,
x,
y0,
Lyy=1,
r2=functions.dummy(),
rtol=1e-9,
maxit=100000,
verbosity='NONE'):
"""
Solve min_y f(x,y) + r_2(y)
Input
-----
x - array
y0 - initial iterate
Lyy - Lipschitz constant of f
r2 - pyunlocbox function object
rtol - stopping criterion for solver
maxit - max iteration count for inner solve
verbosity - level of verbosity for inner solver
Output
------
y - array
"""
# get pyunlocbox function object for f(x,.)
f = self.misfit(x=x)
# setup solver
accel = acceleration.fista_backtracking()
solver = solvers.forward_backward(step=1 / Lyy, accel=accel)
# run solver
results = solvers.solve([f, r2],
y0,
solver,
rtol=rtol,
maxit=maxit,
verbosity=verbosity)
# return result
y = results['sol']
return y
def test_primal_dual_solver_comparison(self):
"""
Test that all primal-dual solvers return the same and correct solution.
I had to create this separate function because the primal-dual solvers
were too slow for the problem above.
"""
# Convex functions.
y = np.array([294, 390, 361])
sol = [1., 1., 1.]
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
f1 = functions.norm_l1(y=y)
f2 = functions.norm_l1()
f3 = functions.dummy()
# Solvers.
step = 0.5 / (1 + np.linalg.norm(L, 2))
slvs = []
slvs.append(solvers.mlfbf(step=step))
slvs.append(solvers.projection_based(step=step))
# Compare solutions.
params = {'rtol': 0, 'verbosity': 'NONE', 'maxit': 50}
niters = [50, 50]
for solver, niter in zip(slvs, niters):
x0 = np.zeros(len(y))
if type(solver) is solvers.mlfbf:
ret = solvers.solve([f1, f2, f3], x0, solver, **params)
else:
ret = solvers.solve([f1, f2], x0, solver, **params)
nptest.assert_allclose(ret['sol'], sol)
self.assertEqual(ret['niter'], niter)
self.assertIs(ret['sol'], x0) # The initial value was modified.
def test_primal_dual_solver_comparison(self):
"""
Test that all primal-dual solvers return the same and correct solution.
I had to create this separate function because the primal-dual solvers
were too slow for the problem above.
"""
# Convex functions.
y = np.array([294, 390, 361])
sol = [1., 1., 1.]
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
f1 = functions.norm_l1(y=y)
f2 = functions.norm_l1()
f3 = functions.dummy()
# Solvers.
step = 0.5 / (1 + np.linalg.norm(L, 2))
slvs = []
slvs.append(solvers.mlfbf(step=step))
slvs.append(solvers.projection_based(step=step))
# Compare solutions.
params = {'rtol': 0, 'verbosity': 'NONE', 'maxit': 50}
niters = [50, 50]
for solver, niter in zip(slvs, niters):
x0 = np.zeros(len(y))
if type(solver) is solvers.mlfbf:
ret = solvers.solve([f1, f2, f3], x0, solver, **params)
else:
ret = solvers.solve([f1, f2], x0, solver, **params)
nptest.assert_allclose(ret['sol'], sol)
self.assertEqual(ret['niter'], niter)
self.assertIs(ret['sol'], x0) # The initial value was modified.
def graph_pnorm_interpolation(self, gradient, P, w, labels_bin, x0=None, p=1., **kwargs):
r"""
Solve an interpolation problem via gradient p-norm minimization.
A signal :math:`x` is estimated from its measurements :math:`y = A(x)` by solving
:math:`\text{arg}\underset{z \in \mathbb{R}^n}{\min}
\| \nabla_G z \|_p^p \text{ subject to } Az = y`
via a primal-dual, forward-backward-forward algorithm.
Parameters
----------
gradient : array_like
A matrix representing the graph gradient operator
P : callable
Orthogonal projection operator mapping points in :math:`z \in \mathbb{R}^n`
onto the set satisfying :math:`A P(z) = A z`.
x0 : array_like, optional
Initial point of the iteration. Must be of dimension n.
(Default is `numpy.random.randn(n)`)
p : {1., 2.}
labels_bin : array_like
A vector that holds the binary labels.
kwargs :
Additional solver parameters, such as maximum number of iterations
(maxit), relative tolerance on the objective (rtol), and verbosity
level (verbosity). See :func:`pyunlocbox.solvers.solve` for the full
list of options.
Returns
-------
x : array_like
The solution to the optimization problem.
"""
grad = lambda z: gradient.dot(z)
div = lambda z: gradient.transpose().dot(z)
# Indicator function of the set satisfying :math:`y = A(z)`
f = functions.func()
f._eval = lambda z: 0
f._prox = lambda z, gamma: P(z, w, labels_bin)
# :math:`\ell_1` norm of the dual variable :math:`d = \nabla_G z`
g = functions.func()
g._eval = lambda z: np.sum(np.abs(grad(z)))
g._prox = lambda d, gamma: functions._soft_threshold(d, gamma)
# :math:`\ell_2` norm of the gradient (for the smooth case)
h = functions.norm_l2(A=grad, At=div)
stepsize = (0.9 / (1. + scipy.sparse.linalg.norm(gradient, ord='fro'))) ** p
solver = solvers.mlfbf(L=grad, Lt=div, step=stepsize)
if p == 1.:
problem = solvers.solve([f, g, functions.dummy()], x0=x0, solver=solver, **kwargs)
return problem['sol']
if p == 2.:
problem = solvers.solve([f, functions.dummy(), h], x0=x0, solver=solver, **kwargs)
return problem['sol']
else:
return x0
def _solve(functions,
x0,
solver=None,
atol=None,
dtol=None,
rtol=1e-3,
xtol=None,
maxit=200,
verbosity='LOW'):
r"""
Solve an optimization problem whose objective function is the sum of some
convex functions.
This function minimizes the objective function :math:`f(x) =
\sum\limits_{k=0}^{k=K} f_k(x)`, i.e. solves
:math:`\operatorname{arg\,min}\limits_x f(x)` for :math:`x \in
\mathbb{R}^{n \times N}` where :math:`n` is the dimensionality of the data
and :math:`N` the number of independent problems. It returns a dictionary
with the found solution and some information about the algorithm
execution.
Note
----
This code is taken from pyunlocbox. Our goal is to modify the function to also
return intermediate solutions.
Parameters
----------
functions : list of objects
A list of convex functions to minimize. These are objects who must
implement the :meth:`pyunlocbox.functions.func.eval` method. The
:meth:`pyunlocbox.functions.func.grad` and / or
:meth:`pyunlocbox.functions.func.prox` methods are required by some
solvers. Note also that some solvers can only handle two convex
functions while others may handle more. Please refer to the
documentation of the considered solver.
x0 : array_like
Starting point of the algorithm, :math:`x_0 \in \mathbb{R}^{n \times
N}`. Note that if you pass a numpy array it will be modified in place
during execution to save memory. It will then contain the solution. Be
careful to pass data of the type (int, float32, float64) you want your
computations to use.
solver : solver class instance, optional
The solver algorithm. It is an object who must inherit from
:class:`pyunlocbox.solvers.solver` and implement the :meth:`_pre`,
:meth:`_algo` and :meth:`_post` methods. If no solver object are
provided, a standard one will be chosen given the number of convex
function objects and their implemented methods.
atol : float, optional
The absolute tolerance stopping criterion. The algorithm stops when
:math:`f(x^t) < atol` where :math:`f(x^t)` is the objective function at
iteration :math:`t`. Default is None.
dtol : float, optional
Stop when the objective function is stable enough, i.e. when
:math:`\left|f(x^t) - f(x^{t-1})\right| < dtol`. Default is None.
rtol : float, optional
The relative tolerance stopping criterion. The algorithm stops when
:math:`\left|\frac{ f(x^t) - f(x^{t-1}) }{ f(x^t) }\right| < rtol`.
Default is :math:`10^{-3}`.
xtol : float, optional
Stop when the variable is stable enough, i.e. when :math:`\frac{\|x^t -
x^{t-1}\|_2}{\sqrt{n N}} < xtol`. Note that additional memory will be
used to store :math:`x^{t-1}`. Default is None.
maxit : int, optional
The maximum number of iterations. Default is 200.
verbosity : {'NONE', 'LOW', 'HIGH', 'ALL'}, optional
The log level : ``'NONE'`` for no log, ``'LOW'`` for resume at
convergence, ``'HIGH'`` for info at all solving steps, ``'ALL'`` for
all possible outputs, including at each steps of the proximal operators
computation. Default is ``'LOW'``.
Returns
-------
sol : ndarray
The problem solution.
solver : str
The used solver.
crit : {'ATOL', 'DTOL', 'RTOL', 'XTOL', 'MAXIT'}
The used stopping criterion. See above for definitions.
niter : int
The number of iterations.
time : float
The execution time in seconds.
objective : ndarray
The successive evaluations of the objective function at each iteration.
backtrace : ndarray
(N_iter + 1, len(sol)) past values of solution.
"""
if verbosity not in ['NONE', 'LOW', 'HIGH', 'ALL']:
raise ValueError('Verbosity should be either NONE, LOW, HIGH or ALL.')
# Add a second dummy convex function if only one function is provided.
if len(functions) < 1:
raise ValueError('At least 1 convex function should be provided.')
elif len(functions) == 1:
functions.append(dummy())
if verbosity in ['LOW', 'HIGH', 'ALL']:
print('INFO: Dummy objective function added.')
# Choose a solver if none provided.
if not solver:
if len(functions) == 2:
fb0 = 'GRAD' in functions[0].cap(x0) and \
'PROX' in functions[1].cap(x0)
fb1 = 'GRAD' in functions[1].cap(x0) and \
'PROX' in functions[0].cap(x0)
dg0 = 'PROX' in functions[0].cap(x0) and \
'PROX' in functions[1].cap(x0)
if fb0 or fb1:
solver = forward_backward() # Need one prox and 1 grad.
elif dg0:
solver = douglas_rachford() # Need two prox.
else:
raise ValueError('No suitable solver for the given functions.')
elif len(functions) > 2:
solver = generalized_forward_backward()
if verbosity in ['LOW', 'HIGH', 'ALL']:
name = solver.__class__.__name__
print('INFO: Selected solver: {}'.format(name))
# Set solver and functions verbosity.
translation = {'ALL': 'HIGH', 'HIGH': 'HIGH', 'LOW': 'LOW', 'NONE': 'NONE'}
solver.verbosity = translation[verbosity]
translation = {'ALL': 'HIGH', 'HIGH': 'LOW', 'LOW': 'NONE', 'NONE': 'NONE'}
functions_verbosity = []
for f in functions:
functions_verbosity.append(f.verbosity)
f.verbosity = translation[verbosity]
tstart = time.time()
crit = None
niter = 0
objective = [[f.eval(x0) for f in functions]]
rtol_only_zeros = True
# Solver specific initialization.
solver.pre(functions, x0)
tape_buffer = np.zeros((1000, len(x0)))
tape_buffer[0] = x0
while not crit:
niter += 1
if xtol is not None:
last_sol = np.array(solver.sol, copy=True)
if verbosity in ['HIGH', 'ALL']:
name = solver.__class__.__name__
print('Iteration {} of {}:'.format(niter, name))
# Solver iterative algorithm.
solver.algo(objective, niter)
tape_buffer[niter] = solver.sol
objective.append([f.eval(solver.sol) for f in functions])
current = np.sum(objective[-1])
last = np.sum(objective[-2])
# Verify stopping criteria.
if atol is not None and current < atol:
crit = 'ATOL'
if dtol is not None and np.abs(current - last) < dtol:
crit = 'DTOL'
if rtol is not None:
div = current # Prevent division by 0.
if div == 0:
if verbosity in ['LOW', 'HIGH', 'ALL']:
print('WARNING: (rtol) objective function is equal to 0 !')
if last != 0:
div = last
else:
div = 1.0 # Result will be zero anyway.
else:
rtol_only_zeros = False
relative = np.abs((current - last) / div)
if relative < rtol and not rtol_only_zeros:
crit = 'RTOL'
if xtol is not None:
err = np.linalg.norm(solver.sol - last_sol)
err /= np.sqrt(last_sol.size)
if err < xtol:
crit = 'XTOL'
if maxit is not None and niter >= maxit:
crit = 'MAXIT'
if verbosity in ['HIGH', 'ALL']:
print(' objective = {:.2e}'.format(current))
# Restore verbosity for functions. In case they are called outside solve().
for k, f in enumerate(functions):
f.verbosity = functions_verbosity[k]
if verbosity in ['LOW', 'HIGH', 'ALL']:
print('Solution found after {} iterations:'.format(niter))
print(' objective function f(sol) = {:e}'.format(current))
print(' stopping criterion: {}'.format(crit))
# Returned dictionary.
result = {
'sol': solver.sol,
'solver': solver.__class__.__name__, # algo for consistency ?
'crit': crit,
'niter': niter,
'time': time.time() - tstart,
'objective': objective
}
try:
# Update dictionary for primal-dual solvers
result['dual_sol'] = solver.dual_sol
except AttributeError:
pass
# Solver specific post-processing (e.g. delete references).
solver.post()
result['backtrace'] = tape_buffer[:(niter + 1)]
return result
def test_solve(self):
"""
Test some features of the solving function.
"""
# We have to set a seed here for the random draw if we are required
# below to assert that the number of iterations of the solvers are
# equal to some specific values. Otherwise, we get trivial errors when
# x0 is a little farther away from y in a given draw.
rs = np.random.RandomState(42)
y = 5 - 10 * rs.uniform(size=(15, 4))
def x0():
return np.zeros(y.shape)
nverb = {'verbosity': 'NONE'}
# Function verbosity.
f = functions.dummy()
self.assertEqual(f.verbosity, 'NONE')
f.verbosity = 'LOW'
solvers.solve([f], x0(), **nverb)
self.assertEqual(f.verbosity, 'LOW')
# Input parameters.
self.assertRaises(ValueError, solvers.solve, [f], x0(), verbosity='??')
# Addition of dummy function.
self.assertRaises(ValueError, solvers.solve, [], x0(), **nverb)
solver = solvers.forward_backward()
solvers.solve([f], x0(), solver, **nverb)
# self.assertIsInstance(solver.f1, functions.dummy)
# self.assertIsInstance(solver.f2, functions.dummy)
# Automatic solver selection.
f0 = functions.func()
f0._eval = lambda x: 0
f0._grad = lambda x: x
f1 = functions.func()
f1._eval = lambda x: 0
f1._grad = lambda x: x
f1._prox = lambda x, T: x
f2 = functions.func()
f2._eval = lambda x: 0
f2._prox = lambda x, T: x
self.assertRaises(ValueError, solvers.solve, [f0, f0], x0(), **nverb)
ret = solvers.solve([f0, f1], x0(), **nverb)
self.assertEqual(ret['solver'], 'forward_backward')
ret = solvers.solve([f1, f0], x0(), **nverb)
self.assertEqual(ret['solver'], 'forward_backward')
ret = solvers.solve([f1, f2], x0(), **nverb)
self.assertEqual(ret['solver'], 'forward_backward')
ret = solvers.solve([f2, f2], x0(), **nverb)
self.assertEqual(ret['solver'], 'douglas_rachford')
ret = solvers.solve([f1, f2, f0], x0(), **nverb)
self.assertEqual(ret['solver'], 'generalized_forward_backward')
# Stopping criteria.
f = functions.norm_l2(y=y)
tol = 1e-6
r = solvers.solve([f], x0(), None, tol, None, None, None, None, 'NONE')
self.assertEqual(r['crit'], 'ATOL')
self.assertLess(np.sum(r['objective'][-1]), tol)
self.assertEqual(r['niter'], 9)
tol = 1e-8
r = solvers.solve([f], x0(), None, None, tol, None, None, None, 'NONE')
self.assertEqual(r['crit'], 'DTOL')
err = np.abs(np.sum(r['objective'][-1]) - np.sum(r['objective'][-2]))
self.assertLess(err, tol)
self.assertEqual(r['niter'], 17)
tol = .1
r = solvers.solve([f], x0(), None, None, None, tol, None, None, 'NONE')
self.assertEqual(r['crit'], 'RTOL')
err = np.abs(np.sum(r['objective'][-1]) - np.sum(r['objective'][-2]))
err /= np.sum(r['objective'][-1])
self.assertLess(err, tol)
self.assertEqual(r['niter'], 13)
tol = 1e-4
r = solvers.solve([f], x0(), None, None, None, None, tol, None, 'NONE')
self.assertEqual(r['crit'], 'XTOL')
r2 = solvers.solve([f], x0(), maxit=r['niter'] - 1, **nverb)
err = np.linalg.norm(r['sol'] - r2['sol']) / np.sqrt(x0().size)
self.assertLess(err, tol)
self.assertEqual(r['niter'], 14)
nit = 15
r = solvers.solve([f], x0(), None, None, None, None, None, nit, 'NONE')
self.assertEqual(r['crit'], 'MAXIT')
self.assertEqual(r['niter'], nit)
# Return values.
f = functions.norm_l2(y=y)
ret = solvers.solve([f], x0(), **nverb)
self.assertEqual(len(ret), 6)
self.assertIsInstance(ret['sol'], np.ndarray)
self.assertIsInstance(ret['solver'], str)
self.assertIsInstance(ret['crit'], str)
self.assertIsInstance(ret['niter'], int)
self.assertIsInstance(ret['time'], float)
self.assertIsInstance(ret['objective'], list)
def test_solve(self):
"""
Test some features of the solving function.
"""
# We have to set a seed here for the random draw if we are required
# below to assert that the number of iterations of the solvers are
# equal to some specific values. Otherwise, we get trivial errors when
# x0 is a little farther away from y in a given draw.
rs = np.random.RandomState(42)
y = 5 - 10 * rs.uniform(size=(15, 4))
def x0(): return np.zeros(y.shape)
nverb = {'verbosity': 'NONE'}
# Function verbosity.
f = functions.dummy()
self.assertEqual(f.verbosity, 'NONE')
f.verbosity = 'LOW'
solvers.solve([f], x0(), **nverb)
self.assertEqual(f.verbosity, 'LOW')
# Input parameters.
self.assertRaises(ValueError, solvers.solve, [f], x0(), verbosity='??')
# Addition of dummy function.
self.assertRaises(ValueError, solvers.solve, [], x0(), **nverb)
solver = solvers.forward_backward()
solvers.solve([f], x0(), solver, **nverb)
# self.assertIsInstance(solver.f1, functions.dummy)
# self.assertIsInstance(solver.f2, functions.dummy)
# Automatic solver selection.
f0 = functions.func()
f0._eval = lambda x: 0
f0._grad = lambda x: x
f1 = functions.func()
f1._eval = lambda x: 0
f1._grad = lambda x: x
f1._prox = lambda x, T: x
f2 = functions.func()
f2._eval = lambda x: 0
f2._prox = lambda x, T: x
self.assertRaises(ValueError, solvers.solve, [f0, f0], x0(), **nverb)
ret = solvers.solve([f0, f1], x0(), **nverb)
self.assertEqual(ret['solver'], 'forward_backward')
ret = solvers.solve([f1, f0], x0(), **nverb)
self.assertEqual(ret['solver'], 'forward_backward')
ret = solvers.solve([f1, f2], x0(), **nverb)
self.assertEqual(ret['solver'], 'forward_backward')
ret = solvers.solve([f2, f2], x0(), **nverb)
self.assertEqual(ret['solver'], 'douglas_rachford')
ret = solvers.solve([f1, f2, f0], x0(), **nverb)
self.assertEqual(ret['solver'], 'generalized_forward_backward')
# Stopping criteria.
f = functions.norm_l2(y=y)
tol = 1e-6
r = solvers.solve([f], x0(), None, tol, None, None, None, None, 'NONE')
self.assertEqual(r['crit'], 'ATOL')
self.assertLess(np.sum(r['objective'][-1]), tol)
self.assertEqual(r['niter'], 9)
tol = 1e-8
r = solvers.solve([f], x0(), None, None, tol, None, None, None, 'NONE')
self.assertEqual(r['crit'], 'DTOL')
err = np.abs(np.sum(r['objective'][-1]) - np.sum(r['objective'][-2]))
self.assertLess(err, tol)
self.assertEqual(r['niter'], 17)
tol = .1
r = solvers.solve([f], x0(), None, None, None, tol, None, None, 'NONE')
self.assertEqual(r['crit'], 'RTOL')
err = np.abs(np.sum(r['objective'][-1]) - np.sum(r['objective'][-2]))
err /= np.sum(r['objective'][-1])
self.assertLess(err, tol)
self.assertEqual(r['niter'], 13)
tol = 1e-4
r = solvers.solve([f], x0(), None, None, None, None, tol, None, 'NONE')
self.assertEqual(r['crit'], 'XTOL')
r2 = solvers.solve([f], x0(), maxit=r['niter'] - 1, **nverb)
err = np.linalg.norm(r['sol'] - r2['sol']) / np.sqrt(x0().size)
self.assertLess(err, tol)
self.assertEqual(r['niter'], 14)
nit = 15
r = solvers.solve([f], x0(), None, None, None, None, None, nit, 'NONE')
self.assertEqual(r['crit'], 'MAXIT')
self.assertEqual(r['niter'], nit)
# Return values.
f = functions.norm_l2(y=y)
ret = solvers.solve([f], x0(), **nverb)
self.assertEqual(len(ret), 6)
self.assertIsInstance(ret['sol'], np.ndarray)
self.assertIsInstance(ret['solver'], str)
self.assertIsInstance(ret['crit'], str)
self.assertIsInstance(ret['niter'], int)
self.assertIsInstance(ret['time'], float)
self.assertIsInstance(ret['objective'], list)
def solve(functions, x0, solver=None, atol=None, dtol=None, rtol=1e-3,
xtol=None, maxit=200, verbosity='LOW'):
r"""
Solve an optimization problem whose objective function is the sum of some
convex functions.
This function minimizes the objective function :math:`f(x) =
\sum\limits_{k=0}^{k=K} f_k(x)`, i.e. solves
:math:`\operatorname{arg\,min}\limits_x f(x)` for :math:`x \in
\mathbb{R}^{n \times N}` where :math:`n` is the dimensionality of the data
and :math:`N` the number of independent problems. It returns a dictionary
with the found solution and some informations about the algorithm
execution.
Parameters
----------
functions : list of objects
A list of convex functions to minimize. These are objects who must
implement the :meth:`pyunlocbox.functions.func.eval` method. The
:meth:`pyunlocbox.functions.func.grad` and / or
:meth:`pyunlocbox.functions.func.prox` methods are required by some
solvers. Note also that some solvers can only handle two convex
functions while others may handle more. Please refer to the
documentation of the considered solver.
x0 : array_like
Starting point of the algorithm, :math:`x_0 \in \mathbb{R}^{n \times
N}`. Note that if you pass a numpy array it will be modified in place
during execution to save memory. It will then contain the solution. Be
careful to pass data of the type (int, float32, float64) you want your
computations to use.
solver : solver class instance, optional
The solver algorithm. It is an object who must inherit from
:class:`pyunlocbox.solvers.solver` and implement the :meth:`_pre`,
:meth:`_algo` and :meth:`_post` methods. If no solver object are
provided, a standard one will be chosen given the number of convex
function objects and their implemented methods.
atol : float, optional
The absolute tolerance stopping criterion. The algorithm stops when
:math:`f(x^t) < atol` where :math:`f(x^t)` is the objective function at
iteration :math:`t`. Default is None.
dtol : float, optional
Stop when the objective function is stable enough, i.e. when
:math:`\left|f(x^t) - f(x^{t-1})\right| < dtol`. Default is None.
rtol : float, optional
The relative tolerance stopping criterion. The algorithm stops when
:math:`\left|\frac{ f(x^t) - f(x^{t-1}) }{ f(x^t) }\right| < rtol`.
Default is :math:`10^{-3}`.
xtol : float, optional
Stop when the variable is stable enough, i.e. when :math:`\frac{\|x^t -
x^{t-1}\|_2}{\sqrt{n N}} < xtol`. Note that additional memory will be
used to store :math:`x^{t-1}`. Default is None.
maxit : int, optional
The maximum number of iterations. Default is 200.
verbosity : {'NONE', 'LOW', 'HIGH', 'ALL'}, optional
The log level : ``'NONE'`` for no log, ``'LOW'`` for resume at
convergence, ``'HIGH'`` for info at all solving steps, ``'ALL'`` for
all possible outputs, including at each steps of the proximal operators
computation. Default is ``'LOW'``.
Returns
-------
sol : ndarray
The problem solution.
solver : str
The used solver.
crit : {'ATOL', 'DTOL', 'RTOL', 'XTOL', 'MAXIT'}
The used stopping criterion. See above for definitions.
niter : int
The number of iterations.
time : float
The execution time in seconds.
objective : ndarray
The successive evaluations of the objective function at each iteration.
Examples
--------
>>> import numpy as np
>>> from pyunlocbox import functions, solvers
Define a problem:
>>> y = [4, 5, 6, 7]
>>> f = functions.norm_l2(y=y)
Solve it:
>>> x0 = np.zeros(len(y))
>>> ret = solvers.solve([f], x0, atol=1e-2, verbosity='ALL')
INFO: Dummy objective function added.
INFO: Selected solver: forward_backward
norm_l2 evaluation: 1.260000e+02
dummy evaluation: 0.000000e+00
INFO: Forward-backward method
Iteration 1 of forward_backward:
norm_l2 evaluation: 1.400000e+01
dummy evaluation: 0.000000e+00
objective = 1.40e+01
Iteration 2 of forward_backward:
norm_l2 evaluation: 2.963739e-01
dummy evaluation: 0.000000e+00
objective = 2.96e-01
Iteration 3 of forward_backward:
norm_l2 evaluation: 7.902529e-02
dummy evaluation: 0.000000e+00
objective = 7.90e-02
Iteration 4 of forward_backward:
norm_l2 evaluation: 5.752265e-02
dummy evaluation: 0.000000e+00
objective = 5.75e-02
Iteration 5 of forward_backward:
norm_l2 evaluation: 5.142032e-03
dummy evaluation: 0.000000e+00
objective = 5.14e-03
Solution found after 5 iterations:
objective function f(sol) = 5.142032e-03
stopping criterion: ATOL
Verify the stopping criterion (should be smaller than atol=1e-2):
>>> np.linalg.norm(ret['sol'] - y)**2 # doctest:+ELLIPSIS
0.00514203...
Show the solution (should be close to y w.r.t. the L2-norm measure):
>>> ret['sol']
array([4.02555301, 5.03194126, 6.03832952, 7.04471777])
Show the used solver:
>>> ret['solver']
'forward_backward'
Show some information about the convergence:
>>> ret['crit']
'ATOL'
>>> ret['niter']
5
>>> ret['time'] # doctest:+SKIP
0.0012578964233398438
>>> ret['objective'] # doctest:+NORMALIZE_WHITESPACE,+ELLIPSIS
[[126.0, 0], [13.99999999..., 0], [0.29637392..., 0], [0.07902528..., 0],
[0.05752265..., 0], [0.00514203..., 0]]
"""
if verbosity not in ['NONE', 'LOW', 'HIGH', 'ALL']:
raise ValueError('Verbosity should be either NONE, LOW, HIGH or ALL.')
# Add a second dummy convex function if only one function is provided.
if len(functions) < 1:
raise ValueError('At least 1 convex function should be provided.')
elif len(functions) == 1:
functions.append(dummy())
if verbosity in ['LOW', 'HIGH', 'ALL']:
print('INFO: Dummy objective function added.')
# Choose a solver if none provided.
if not solver:
if len(functions) == 2:
fb0 = 'GRAD' in functions[0].cap(x0) and \
'PROX' in functions[1].cap(x0)
fb1 = 'GRAD' in functions[1].cap(x0) and \
'PROX' in functions[0].cap(x0)
dg0 = 'PROX' in functions[0].cap(x0) and \
'PROX' in functions[1].cap(x0)
if fb0 or fb1:
solver = forward_backward() # Need one prox and 1 grad.
elif dg0:
solver = douglas_rachford() # Need two prox.
else:
raise ValueError('No suitable solver for the given functions.')
elif len(functions) > 2:
solver = generalized_forward_backward()
if verbosity in ['LOW', 'HIGH', 'ALL']:
name = solver.__class__.__name__
print('INFO: Selected solver: {}'.format(name))
# Set solver and functions verbosity.
translation = {'ALL': 'HIGH', 'HIGH': 'HIGH', 'LOW': 'LOW', 'NONE': 'NONE'}
solver.verbosity = translation[verbosity]
translation = {'ALL': 'HIGH', 'HIGH': 'LOW', 'LOW': 'NONE', 'NONE': 'NONE'}
functions_verbosity = []
for f in functions:
functions_verbosity.append(f.verbosity)
f.verbosity = translation[verbosity]
tstart = time.time()
crit = None
niter = 0
objective = [[f.eval(x0) for f in functions]]
rtol_only_zeros = True
# Solver specific initialization.
solver.pre(functions, x0)
while not crit:
niter += 1
if xtol is not None:
last_sol = np.array(solver.sol, copy=True)
if verbosity in ['HIGH', 'ALL']:
name = solver.__class__.__name__
print('Iteration {} of {}:'.format(niter, name))
# Solver iterative algorithm.
solver.algo(objective, niter)
objective.append([f.eval(solver.sol) for f in functions])
current = np.sum(objective[-1])
last = np.sum(objective[-2])
# Verify stopping criteria.
if atol is not None and current < atol:
crit = 'ATOL'
if dtol is not None and np.abs(current - last) < dtol:
crit = 'DTOL'
if rtol is not None:
div = current # Prevent division by 0.
if div == 0:
if verbosity in ['LOW', 'HIGH', 'ALL']:
print('WARNING: (rtol) objective function is equal to 0 !')
if last != 0:
div = last
else:
div = 1.0 # Result will be zero anyway.
else:
rtol_only_zeros = False
relative = np.abs((current - last) / div)
if relative < rtol and not rtol_only_zeros:
crit = 'RTOL'
if xtol is not None:
err = np.linalg.norm(solver.sol - last_sol)
err /= np.sqrt(last_sol.size)
if err < xtol:
crit = 'XTOL'
if maxit is not None and niter >= maxit:
crit = 'MAXIT'
if verbosity in ['HIGH', 'ALL']:
print(' objective = {:.2e}'.format(current))
# Restore verbosity for functions. In case they are called outside solve().
for k, f in enumerate(functions):
f.verbosity = functions_verbosity[k]
if verbosity in ['LOW', 'HIGH', 'ALL']:
print('Solution found after {} iterations:'.format(niter))
print(' objective function f(sol) = {:e}'.format(current))
print(' stopping criterion: {}'.format(crit))
# Returned dictionary.
result = {'sol': solver.sol,
'solver': solver.__class__.__name__, # algo for consistency ?
'crit': crit,
'niter': niter,
'time': time.time() - tstart,
'objective': objective}
try:
# Update dictionary for primal-dual solvers
result['dual_sol'] = solver.dual_sol
except AttributeError:
pass
# Solver specific post-processing (e.g. delete references).
solver.post()
return result
def test_mlfbf(self):
"""
Test the MLFBF solver with arbitrarily selected functions.
"""
x = [1., 1., 1.]
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
max_step = 1 / (1 + np.linalg.norm(L, 2))
solver = solvers.mlfbf(L=L, step=max_step / 2.)
params = {'solver': solver, 'verbosity': 'NONE'}
def x0():
return np.zeros(len(x))
# L2-norm prox and dummy prox.
f = functions.dummy()
f._prox = lambda x, T: np.maximum(np.zeros(len(x)), x)
g = functions.norm_l2(lambda_=0.5)
h = functions.norm_l2(y=np.array([294, 390, 361]), lambda_=0.5)
ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Same test, but with callable L
solver = solvers.mlfbf(L=lambda x: np.dot(L, x),
Lt=lambda y: np.dot(L.T, y),
d0=np.dot(L, x0()),
step=max_step / 2.)
ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Sanity check
self.assertRaises(ValueError, solver.pre, [f, g], x0())
# Make a second test where the solution is calculated by hand
n = 10
y = np.random.rand(n) * 2
z = np.random.rand(n)
c = 1
delta = (y - z - c)**2 + 4 * (1 + y * z - z * c)
sol = 0.5 * ((y - z - c) + np.sqrt(delta))
class mlog(functions.func):
def __init__(self, z):
super().__init__()
self.z = z
def _eval(self, x):
return -np.sum(np.log(x + self.z))
def _prox(self, x, T):
delta = (x - self.z)**2 + 4 * (T + x * self.z)
sol = 0.5 * (x - self.z + np.sqrt(delta))
return sol
f = functions.norm_l1(lambda_=c)
g = mlog(z=z)
h = functions.norm_l2(lambda_=0.5, y=y)
mu = 1 + 1
step = 1 / mu / 2
solver = solvers.mlfbf(step=step)
ret = solvers.solve([f, g, h],
y.copy(),
solver,
maxit=200,
rtol=0,
verbosity="NONE")
nptest.assert_allclose(ret["sol"], sol, atol=1e-10)
# Make a final test where the function g can not be evaluate
# on the primal variables
y = np.random.rand(3)
y_2 = L.dot(y)
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
x0 = np.zeros(len(y))
f = functions.norm_l1(y=y)
g = functions.norm_l2(lambda_=0.5, y=y_2)
h = functions.norm_l2(y=y, lambda_=0.5)
max_step = 1 / (1 + np.linalg.norm(L, 2))
solver = solvers.mlfbf(L=L, step=max_step / 2.)
ret = solvers.solve([f, g, h], x0, solver, maxit=1000, rtol=0)
np.testing.assert_allclose(ret["sol"], y)
def solve(functions,
x0,
solver=None,
atol=None,
dtol=None,
rtol=1e-3,
xtol=None,
maxit=200,
verbosity='LOW'):
r"""
Solve an optimization problem whose objective function is the sum of some
convex functions.
This function minimizes the objective function :math:`f(x) =
\sum\limits_{k=0}^{k=K} f_k(x)`, i.e. solves
:math:`\operatorname{arg\,min}\limits_x f(x)` for :math:`x \in
\mathbb{R}^{n \times N}` where :math:`n` is the dimensionality of the data
and :math:`N` the number of independent problems. It returns a dictionary
with the found solution and some informations about the algorithm
execution.
Parameters
----------
functions : list of objects
A list of convex functions to minimize. These are objects who must
implement the :meth:`pyunlocbox.functions.func.eval` method. The
:meth:`pyunlocbox.functions.func.grad` and / or
:meth:`pyunlocbox.functions.func.prox` methods are required by some
solvers. Note also that some solvers can only handle two convex
functions while others may handle more. Please refer to the
documentation of the considered solver.
x0 : array_like
Starting point of the algorithm, :math:`x_0 \in \mathbb{R}^{n \times
N}`. Note that if you pass a numpy array it will be modified in place
during execution to save memory. It will then contain the solution. Be
careful to pass data of the type (int, float32, float64) you want your
computations to use.
solver : solver class instance, optional
The solver algorithm. It is an object who must inherit from
:class:`pyunlocbox.solvers.solver` and implement the :meth:`_pre`,
:meth:`_algo` and :meth:`_post` methods. If no solver object are
provided, a standard one will be chosen given the number of convex
function objects and their implemented methods.
atol : float, optional
The absolute tolerance stopping criterion. The algorithm stops when
:math:`f(x^t) < atol` where :math:`f(x^t)` is the objective function at
iteration :math:`t`. Default is None.
dtol : float, optional
Stop when the objective function is stable enough, i.e. when
:math:`\left|f(x^t) - f(x^{t-1})\right| < dtol`. Default is None.
rtol : float, optional
The relative tolerance stopping criterion. The algorithm stops when
:math:`\left|\frac{ f(x^t) - f(x^{t-1}) }{ f(x^t) }\right| < rtol`.
Default is :math:`10^{-3}`.
xtol : float, optional
Stop when the variable is stable enough, i.e. when :math:`\frac{\|x^t -
x^{t-1}\|_2}{\sqrt{n N}} < xtol`. Note that additional memory will be
used to store :math:`x^{t-1}`. Default is None.
maxit : int, optional
The maximum number of iterations. Default is 200.
verbosity : {'NONE', 'LOW', 'HIGH', 'ALL'}, optional
The log level : ``'NONE'`` for no log, ``'LOW'`` for resume at
convergence, ``'HIGH'`` for info at all solving steps, ``'ALL'`` for
all possible outputs, including at each steps of the proximal operators
computation. Default is ``'LOW'``.
Returns
-------
sol : ndarray
The problem solution.
solver : str
The used solver.
crit : {'ATOL', 'DTOL', 'RTOL', 'XTOL', 'MAXIT'}
The used stopping criterion. See above for definitions.
niter : int
The number of iterations.
time : float
The execution time in seconds.
objective : ndarray
The successive evaluations of the objective function at each iteration.
Examples
--------
>>> import numpy as np
>>> from pyunlocbox import functions, solvers
Define a problem:
>>> y = [4, 5, 6, 7]
>>> f = functions.norm_l2(y=y)
Solve it:
>>> x0 = np.zeros(len(y))
>>> ret = solvers.solve([f], x0, atol=1e-2, verbosity='ALL')
INFO: Dummy objective function added.
INFO: Selected solver: forward_backward
norm_l2 evaluation: 1.260000e+02
dummy evaluation: 0.000000e+00
INFO: Forward-backward method
Iteration 1 of forward_backward:
norm_l2 evaluation: 1.400000e+01
dummy evaluation: 0.000000e+00
objective = 1.40e+01
Iteration 2 of forward_backward:
norm_l2 evaluation: 2.963739e-01
dummy evaluation: 0.000000e+00
objective = 2.96e-01
Iteration 3 of forward_backward:
norm_l2 evaluation: 7.902529e-02
dummy evaluation: 0.000000e+00
objective = 7.90e-02
Iteration 4 of forward_backward:
norm_l2 evaluation: 5.752265e-02
dummy evaluation: 0.000000e+00
objective = 5.75e-02
Iteration 5 of forward_backward:
norm_l2 evaluation: 5.142032e-03
dummy evaluation: 0.000000e+00
objective = 5.14e-03
Solution found after 5 iterations:
objective function f(sol) = 5.142032e-03
stopping criterion: ATOL
Verify the stopping criterion (should be smaller than atol=1e-2):
>>> np.linalg.norm(ret['sol'] - y)**2 # doctest:+ELLIPSIS
0.00514203...
Show the solution (should be close to y w.r.t. the L2-norm measure):
>>> ret['sol']
array([4.02555301, 5.03194126, 6.03832952, 7.04471777])
Show the used solver:
>>> ret['solver']
'forward_backward'
Show some information about the convergence:
>>> ret['crit']
'ATOL'
>>> ret['niter']
5
>>> ret['time'] # doctest:+SKIP
0.0012578964233398438
>>> ret['objective'] # doctest:+NORMALIZE_WHITESPACE,+ELLIPSIS
[[126.0, 0], [13.99999999..., 0], [0.29637392..., 0], [0.07902528..., 0],
[0.05752265..., 0], [0.00514203..., 0]]
"""
if verbosity not in ['NONE', 'LOW', 'HIGH', 'ALL']:
raise ValueError('Verbosity should be either NONE, LOW, HIGH or ALL.')
# Add a second dummy convex function if only one function is provided.
if len(functions) < 1:
raise ValueError('At least 1 convex function should be provided.')
elif len(functions) == 1:
functions.append(dummy())
if verbosity in ['LOW', 'HIGH', 'ALL']:
print('INFO: Dummy objective function added.')
# Choose a solver if none provided.
if not solver:
if len(functions) == 2:
fb0 = 'GRAD' in functions[0].cap(x0) and \
'PROX' in functions[1].cap(x0)
fb1 = 'GRAD' in functions[1].cap(x0) and \
'PROX' in functions[0].cap(x0)
dg0 = 'PROX' in functions[0].cap(x0) and \
'PROX' in functions[1].cap(x0)
if fb0 or fb1:
solver = forward_backward() # Need one prox and 1 grad.
elif dg0:
solver = douglas_rachford() # Need two prox.
else:
raise ValueError('No suitable solver for the given functions.')
elif len(functions) > 2:
solver = generalized_forward_backward()
if verbosity in ['LOW', 'HIGH', 'ALL']:
name = solver.__class__.__name__
print('INFO: Selected solver: {}'.format(name))
# Set solver and functions verbosity.
translation = {'ALL': 'HIGH', 'HIGH': 'HIGH', 'LOW': 'LOW', 'NONE': 'NONE'}
solver.verbosity = translation[verbosity]
translation = {'ALL': 'HIGH', 'HIGH': 'LOW', 'LOW': 'NONE', 'NONE': 'NONE'}
functions_verbosity = []
for f in functions:
functions_verbosity.append(f.verbosity)
f.verbosity = translation[verbosity]
tstart = time.time()
crit = None
niter = 0
objective = [[f.eval(x0) for f in functions]]
rtol_only_zeros = True
# Solver specific initialization.
solver.pre(functions, x0)
while not crit:
niter += 1
if xtol is not None:
last_sol = np.array(solver.sol, copy=True)
if verbosity in ['HIGH', 'ALL']:
name = solver.__class__.__name__
print('Iteration {} of {}:'.format(niter, name))
# Solver iterative algorithm.
solver.algo(objective, niter)
objective.append([f.eval(solver.sol) for f in functions])
current = np.sum(objective[-1])
last = np.sum(objective[-2])
# Verify stopping criteria.
if atol is not None and current < atol:
crit = 'ATOL'
if dtol is not None and np.abs(current - last) < dtol:
crit = 'DTOL'
if rtol is not None:
div = current # Prevent division by 0.
if div == 0:
if verbosity in ['LOW', 'HIGH', 'ALL']:
print('WARNING: (rtol) objective function is equal to 0 !')
if last != 0:
div = last
else:
div = 1.0 # Result will be zero anyway.
else:
rtol_only_zeros = False
relative = np.abs((current - last) / div)
if relative < rtol and not rtol_only_zeros:
crit = 'RTOL'
if xtol is not None:
err = np.linalg.norm(solver.sol - last_sol)
err /= np.sqrt(last_sol.size)
if err < xtol:
crit = 'XTOL'
if maxit is not None and niter >= maxit:
crit = 'MAXIT'
if verbosity in ['HIGH', 'ALL']:
print(' objective = {:.2e}'.format(current))
# Restore verbosity for functions. In case they are called outside solve().
for k, f in enumerate(functions):
f.verbosity = functions_verbosity[k]
if verbosity in ['LOW', 'HIGH', 'ALL']:
print('Solution found after {} iterations:'.format(niter))
print(' objective function f(sol) = {:e}'.format(current))
print(' stopping criterion: {}'.format(crit))
# Returned dictionary.
result = {
'sol': solver.sol,
'solver': solver.__class__.__name__, # algo for consistency ?
'crit': crit,
'niter': niter,
'time': time.time() - tstart,
'objective': objective
}
try:
# Update dictionary for primal-dual solvers
result['dual_sol'] = solver.dual_sol
except AttributeError:
pass
# Solver specific post-processing (e.g. delete references).
solver.post()
return result
def test_forward_backward_fista(self):
"""
Test forward-backward splitting solver with fista acceleration,
solving problems with L1-norm, L2-norm, and dummy functions.
"""
y = [4., 5., 6., 7.]
solver = solvers.forward_backward(accel=acceleration.fista())
param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 60)
# Dummy prox and L2-norm gradient.
f1 = functions.dummy()
f2 = functions.norm_l2(y=y, lambda_=0.6)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 84)
# L2-norm prox and L2-norm gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.norm_l2(y=y)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y, rtol=1e-2)
self.assertEqual(ret['crit'], 'MAXIT')
self.assertEqual(ret['niter'], 200)
# L1-norm prox and dummy gradient.
f1 = functions.norm_l1(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 6)
# Dummy prox and L1-norm gradient. As L1-norm possesses no gradient,
# the algorithm exchanges the functions : exact same solution.
f1 = functions.dummy()
f2 = functions.norm_l1(y=y)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 6)
# L1-norm prox and L1-norm gradient. L1-norm possesses no gradient.
f1 = functions.norm_l1(y=y)
f2 = functions.norm_l1(y=y)
self.assertRaises(ValueError, solvers.solve,
[f1, f2], np.zeros(len(y)), **param)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 10)
def test_forward_backward_fista(self):
"""
Test forward-backward splitting solver with fista acceleration,
solving problems with L1-norm, L2-norm, and dummy functions.
"""
y = [4., 5., 6., 7.]
solver = solvers.forward_backward(accel=acceleration.fista())
param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 60)
# Dummy prox and L2-norm gradient.
f1 = functions.dummy()
f2 = functions.norm_l2(y=y, lambda_=0.6)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 84)
# L2-norm prox and L2-norm gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.norm_l2(y=y)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y, rtol=1e-2)
self.assertEqual(ret['crit'], 'MAXIT')
self.assertEqual(ret['niter'], 200)
# L1-norm prox and dummy gradient.
f1 = functions.norm_l1(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 6)
# Dummy prox and L1-norm gradient. As L1-norm possesses no gradient,
# the algorithm exchanges the functions : exact same solution.
f1 = functions.dummy()
f2 = functions.norm_l1(y=y)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 6)
# L1-norm prox and L1-norm gradient. L1-norm possesses no gradient.
f1 = functions.norm_l1(y=y)
f2 = functions.norm_l1(y=y)
self.assertRaises(ValueError, solvers.solve, [f1, f2],
np.zeros(len(y)), **param)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 10)
