def test_forward_backward(self): """ Test forward-backward splitting algorithm without acceleration, and with L1-norm, L2-norm, and dummy functions. """ y = [4., 5., 6., 7.] solver = solvers.forward_backward(accel=acceleration.dummy()) param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'} # L2-norm prox and dummy gradient. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 35) # L1-norm prox and L2-norm gradient. f1 = functions.norm_l1(y=y, lambda_=1.0) f2 = functions.norm_l2(y=y, lambda_=0.8) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 4) # Sanity check f3 = functions.dummy() x0 = np.zeros((4,)) self.assertRaises(ValueError, solver.pre, [f1, f2, f3], x0)
def test_forward_backward(self): """ Test forward-backward splitting algorithm without acceleration, and with L1-norm, L2-norm, and dummy functions. """ y = [4., 5., 6., 7.] solver = solvers.forward_backward(accel=acceleration.dummy()) param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'} # L2-norm prox and dummy gradient. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 35) # L1-norm prox and L2-norm gradient. f1 = functions.norm_l1(y=y, lambda_=1.0) f2 = functions.norm_l2(y=y, lambda_=0.8) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 4) # Sanity check f3 = functions.dummy() x0 = np.zeros((4, )) self.assertRaises(ValueError, solver.pre, [f1, f2, f3], x0)
def test_solver(self): """ Base solver class. """ funs = [functions.dummy(), functions.dummy()] x0 = np.zeros((4,)) s = solvers.solver() s.sol = x0 self.assertRaises(ValueError, s.__init__, -1.) self.assertRaises(NotImplementedError, s.pre, funs, x0) self.assertRaises(NotImplementedError, s._algo) self.assertRaises(NotImplementedError, s.post)
def test_solver(self): """ Base solver class. """ funs = [functions.dummy(), functions.dummy()] x0 = np.zeros((4, )) s = solvers.solver() s.sol = x0 self.assertRaises(ValueError, s.__init__, -1.) self.assertRaises(NotImplementedError, s.pre, funs, x0) self.assertRaises(NotImplementedError, s._algo) self.assertRaises(NotImplementedError, s.post)
def test_douglas_rachford(self): """ Test douglas-rachford solver with L1-norm, L2-norm and dummy functions. """ y = [4, 5, 6, 7] solver = solvers.douglas_rachford() param = {'solver': solver, 'verbosity': 'NONE'} # L2-norm prox and dummy prox. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 35) # L2-norm prox and L1-norm prox. f1 = functions.norm_l2(y=y) f2 = functions.norm_l1(y=y) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 4) # Sanity checks x0 = np.zeros((4,)) solver.lambda_ = 2. self.assertRaises(ValueError, solver.pre, [f1, f2], x0) solver.lambda_ = -2. self.assertRaises(ValueError, solver.pre, [f1, f2], x0) self.assertRaises(ValueError, solver.pre, [f1, f2, f1], x0)
def test_douglas_rachford(self): """ Test douglas-rachford solver with L1-norm, L2-norm and dummy functions. """ y = [4, 5, 6, 7] solver = solvers.douglas_rachford() param = {'solver': solver, 'verbosity': 'NONE'} # L2-norm prox and dummy prox. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 35) # L2-norm prox and L1-norm prox. f1 = functions.norm_l2(y=y) f2 = functions.norm_l1(y=y) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 4) # Sanity checks x0 = np.zeros((4, )) solver.lambda_ = 2. self.assertRaises(ValueError, solver.pre, [f1, f2], x0) solver.lambda_ = -2. self.assertRaises(ValueError, solver.pre, [f1, f2], x0) self.assertRaises(ValueError, solver.pre, [f1, f2, f1], x0)
def test_mlfbf(self): """ Test the MLFBF solver with arbitrarily selected functions. """ x = [1., 1., 1.] L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]]) max_step = 1 / (1 + np.linalg.norm(L, 2)) solver = solvers.mlfbf(L=L, step=max_step / 2.) params = {'solver': solver, 'verbosity': 'NONE'} def x0(): return np.zeros(len(x)) # L2-norm prox and dummy prox. f = functions.dummy() f._prox = lambda x, T: np.maximum(np.zeros(len(x)), x) g = functions.norm_l2(lambda_=0.5) h = functions.norm_l2(y=np.array([294, 390, 361]), lambda_=0.5) ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params) nptest.assert_allclose(ret['sol'], x, rtol=1e-5) # Same test, but with callable L solver = solvers.mlfbf(L=lambda x: np.dot(L, x), Lt=lambda y: np.dot(L.T, y), d0=np.dot(L, x0()), step=max_step / 2.) ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params) nptest.assert_allclose(ret['sol'], x, rtol=1e-5) # Sanity check self.assertRaises(ValueError, solver.pre, [f, g], x0())
def test_regularized_nonlinear(self): """ Test gradient descent solver with regularized non-linear acceleration, solving problems with L2-norm functions. """ dim = 25 np.random.seed(0) x0 = np.random.rand(dim) xstar = np.random.rand(dim) x0 = xstar + 5. * (x0 - xstar) / np.linalg.norm(x0 - xstar) A = np.random.rand(dim, dim) step = 1 / np.linalg.norm(np.dot(A.T, A)) accel = acceleration.regularized_nonlinear(k=5) solver = solvers.gradient_descent(step=step, accel=accel) param = {'solver': solver, 'rtol': 0, 'maxit': 200, 'verbosity': 'NONE'} # L2-norm prox and dummy gradient. f1 = functions.norm_l2(lambda_=0.5, A=A, y=np.dot(A, xstar)) f2 = functions.dummy() ret = solvers.solve([f1, f2], x0, **param) pctdiff = 100 * np.sum((xstar - ret['sol'])**2) / np.sum(xstar**2) nptest.assert_array_less(pctdiff, 1.91) # Sanity checks accel = acceleration.regularized_nonlinear() self.assertRaises(ValueError, accel.__init__, 10, ['not', 'good']) self.assertRaises(ValueError, accel.__init__, 10, 'nope')
def test_accel(self): """ Test base acceleration scheme class """ funs = [functions.dummy(), functions.dummy()] x0 = np.zeros((4, )) a = acceleration.accel() s = solvers.forward_backward() o = [[1., 2.], [0., 1.]] n = 2 self.assertRaises(NotImplementedError, a.pre, funs, x0) self.assertRaises(NotImplementedError, a.update_step, s, o, n) self.assertRaises(NotImplementedError, a.update_sol, s, o, n) self.assertRaises(NotImplementedError, a.post)
def test_backtracking(self): """ Test forward-backward splitting solver with backtracking, solving problems with L1-norm, L2-norm, and dummy functions. """ # Test constructor sanity a = acceleration.backtracking() self.assertRaises(ValueError, a.__init__, 2.) self.assertRaises(ValueError, a.__init__, -2.) y = [4., 5., 6., 7.] accel = acceleration.backtracking() step = 10 # Make sure backtracking is called solver = solvers.forward_backward(accel=accel, step=step) param = {'solver': solver, 'atol': 1e-32, 'verbosity': 'NONE'} # L2-norm prox and dummy gradient. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'ATOL') self.assertEqual(ret['niter'], 13) # L1-norm prox and L2-norm gradient. f1 = functions.norm_l1(y=y, lambda_=1.0) f2 = functions.norm_l2(y=y, lambda_=0.8) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'ATOL') self.assertEqual(ret['niter'], 4)
def test_forward_backward_fista_backtracking(self): """ Test forward-backward splitting solver with fista acceleration and backtracking, solving problems with L1-norm, L2-norm, and dummy functions. """ y = [4., 5., 6., 7.] accel = acceleration.fista_backtracking() solver = solvers.forward_backward(accel=accel) param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'} # L2-norm prox and dummy gradient. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 60) # L1-norm prox and L2-norm gradient. f1 = functions.norm_l1(y=y, lambda_=1.0) f2 = functions.norm_l2(y=y, lambda_=0.8) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 3)
def test_backtracking(self): """ Test forward-backward splitting solver with backtracking, solving problems with L1-norm, L2-norm, and dummy functions. """ # Test constructor sanity a = acceleration.backtracking() self.assertRaises(ValueError, a.__init__, 2.) self.assertRaises(ValueError, a.__init__, -2.) y = [4., 5., 6., 7.] accel = acceleration.backtracking() step = 10 # Make sure backtracking is called solver = solvers.forward_backward(accel=accel, step=step) param = {'solver': solver, 'atol': 1e-32, 'verbosity': 'NONE'} # L2-norm prox and dummy gradient. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'ATOL') self.assertEqual(ret['niter'], 13) # L1-norm prox and L2-norm gradient. f1 = functions.norm_l1(y=y, lambda_=1.0) f2 = functions.norm_l2(y=y, lambda_=0.8) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'ATOL') self.assertLessEqual(ret['niter'], 4) # win64 takes one iteration
def test_regularized_nonlinear(self): """ Test gradient descent solver with regularized non-linear acceleration, solving problems with L2-norm functions. """ dim = 25 np.random.seed(0) x0 = np.random.rand(dim) xstar = np.random.rand(dim) x0 = xstar + 5. * (x0 - xstar) / np.linalg.norm(x0 - xstar) A = np.random.rand(dim, dim) step = 1 / np.linalg.norm(np.dot(A.T, A)) accel = acceleration.regularized_nonlinear(k=5) solver = solvers.gradient_descent(step=step, accel=accel) param = { 'solver': solver, 'rtol': 0, 'maxit': 200, 'verbosity': 'NONE' } # L2-norm prox and dummy gradient. f1 = functions.norm_l2(lambda_=0.5, A=A, y=np.dot(A, xstar)) f2 = functions.dummy() ret = solvers.solve([f1, f2], x0, **param) pctdiff = 100 * np.sum((xstar - ret['sol'])**2) / np.sum(xstar**2) nptest.assert_array_less(pctdiff, 1.91) # Sanity checks accel = acceleration.regularized_nonlinear() self.assertRaises(ValueError, accel.__init__, 10, ['not', 'good']) self.assertRaises(ValueError, accel.__init__, 10, 'nope')
def test_accel(self): """ Test base acceleration scheme class """ funs = [functions.dummy(), functions.dummy()] x0 = np.zeros((4,)) a = acceleration.accel() s = solvers.forward_backward() o = [[1., 2.], [0., 1.]] n = 2 self.assertRaises(NotImplementedError, a.pre, funs, x0) self.assertRaises(NotImplementedError, a.update_step, s, o, n) self.assertRaises(NotImplementedError, a.update_sol, s, o, n) self.assertRaises(NotImplementedError, a.post)
def test_primal_dual_solver_comparison(self): """ Test that all primal-dual solvers return the same and correct solution. I had to create this separate function because the primal-dual solvers were too slow for the problem above. """ # Convex functions. y = np.random.randn(3) L = np.random.randn(4, 3) sol = y y2 = L.dot(y) f1 = functions.norm_l1(y=y) f2 = functions.norm_l2(y=y2) f3 = functions.dummy() # Solvers. step = 0.5 / (1 + np.linalg.norm(L, 2)) slvs = [] slvs.append(solvers.mlfbf(step=step, L=L)) slvs.append(solvers.projection_based(step=step, L=L)) # Compare solutions. niter = 1000 params = {'rtol': 0, 'verbosity': 'NONE', 'maxit': niter} for solver in slvs: x0 = np.zeros(len(y)) if type(solver) is solvers.mlfbf: ret = solvers.solve([f1, f2, f3], x0, solver, **params) else: ret = solvers.solve([f1, f2], x0, solver, **params) nptest.assert_allclose(ret['sol'], sol) self.assertEqual(ret['niter'], niter) # The initial value was not modified. nptest.assert_array_equal(x0, np.zeros(len(y))) if type(solver) is solvers.mlfbf: ret = solvers.solve([f1, f2, f3], x0, solver, inplace=True, **params) else: ret = solvers.solve([f1, f2], x0, solver, inplace=True, **params) # The initial value was modified. self.assertIs(ret['sol'], x0) nptest.assert_allclose(ret['sol'], sol)
def test_dummy(self): """ Test the dummy derived class. All the methods should return 0. """ f = functions.dummy() self.assertEqual(f.eval(34), 0) nptest.assert_array_equal(f.grad(34), [0]) nptest.assert_array_equal(f.prox(34, 1), [34]) x = [34, 2, 1.0, -10.2] self.assertEqual(f.eval(x), 0) nptest.assert_array_equal(f.grad(x), np.zeros(len(x))) nptest.assert_array_equal(f.prox(x, 1), x)
def test_prox_star(self): n = 10 x = 3 * np.random.randn(n, 1) f = functions.norm_l1() f2 = functions.dummy() f2.prox = lambda x, T: functions._prox_star(f, x, T) gamma = np.random.rand() p1 = f.prox(x, gamma) p2 = functions._prox_star(f2, x, gamma) np.testing.assert_array_almost_equal(p1, p2) p1 = f.prox(x, gamma) - x p2 = -gamma * f2.prox(x / gamma, 1 / gamma) np.testing.assert_array_almost_equal(p1, p2)
def inner_solve(self, x, y0, Lyy=1, r2=functions.dummy(), rtol=1e-9, maxit=100000, verbosity='NONE'): """ Solve min_y f(x,y) + r_2(y) Input ----- x - array y0 - initial iterate Lyy - Lipschitz constant of f r2 - pyunlocbox function object rtol - stopping criterion for solver maxit - max iteration count for inner solve verbosity - level of verbosity for inner solver Output ------ y - array """ # get pyunlocbox function object for f(x,.) f = self.misfit(x=x) # setup solver accel = acceleration.fista_backtracking() solver = solvers.forward_backward(step=1 / Lyy, accel=accel) # run solver results = solvers.solve([f, r2], y0, solver, rtol=rtol, maxit=maxit, verbosity=verbosity) # return result y = results['sol'] return y
def test_primal_dual_solver_comparison(self): """ Test that all primal-dual solvers return the same and correct solution. I had to create this separate function because the primal-dual solvers were too slow for the problem above. """ # Convex functions. y = np.array([294, 390, 361]) sol = [1., 1., 1.] L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]]) f1 = functions.norm_l1(y=y) f2 = functions.norm_l1() f3 = functions.dummy() # Solvers. step = 0.5 / (1 + np.linalg.norm(L, 2)) slvs = [] slvs.append(solvers.mlfbf(step=step)) slvs.append(solvers.projection_based(step=step)) # Compare solutions. params = {'rtol': 0, 'verbosity': 'NONE', 'maxit': 50} niters = [50, 50] for solver, niter in zip(slvs, niters): x0 = np.zeros(len(y)) if type(solver) is solvers.mlfbf: ret = solvers.solve([f1, f2, f3], x0, solver, **params) else: ret = solvers.solve([f1, f2], x0, solver, **params) nptest.assert_allclose(ret['sol'], sol) self.assertEqual(ret['niter'], niter) self.assertIs(ret['sol'], x0) # The initial value was modified.
def graph_pnorm_interpolation(self, gradient, P, w, labels_bin, x0=None, p=1., **kwargs): r""" Solve an interpolation problem via gradient p-norm minimization. A signal :math:`x` is estimated from its measurements :math:`y = A(x)` by solving :math:`\text{arg}\underset{z \in \mathbb{R}^n}{\min} \| \nabla_G z \|_p^p \text{ subject to } Az = y` via a primal-dual, forward-backward-forward algorithm. Parameters ---------- gradient : array_like A matrix representing the graph gradient operator P : callable Orthogonal projection operator mapping points in :math:`z \in \mathbb{R}^n` onto the set satisfying :math:`A P(z) = A z`. x0 : array_like, optional Initial point of the iteration. Must be of dimension n. (Default is `numpy.random.randn(n)`) p : {1., 2.} labels_bin : array_like A vector that holds the binary labels. kwargs : Additional solver parameters, such as maximum number of iterations (maxit), relative tolerance on the objective (rtol), and verbosity level (verbosity). See :func:`pyunlocbox.solvers.solve` for the full list of options. Returns ------- x : array_like The solution to the optimization problem. """ grad = lambda z: gradient.dot(z) div = lambda z: gradient.transpose().dot(z) # Indicator function of the set satisfying :math:`y = A(z)` f = functions.func() f._eval = lambda z: 0 f._prox = lambda z, gamma: P(z, w, labels_bin) # :math:`\ell_1` norm of the dual variable :math:`d = \nabla_G z` g = functions.func() g._eval = lambda z: np.sum(np.abs(grad(z))) g._prox = lambda d, gamma: functions._soft_threshold(d, gamma) # :math:`\ell_2` norm of the gradient (for the smooth case) h = functions.norm_l2(A=grad, At=div) stepsize = (0.9 / (1. + scipy.sparse.linalg.norm(gradient, ord='fro'))) ** p solver = solvers.mlfbf(L=grad, Lt=div, step=stepsize) if p == 1.: problem = solvers.solve([f, g, functions.dummy()], x0=x0, solver=solver, **kwargs) return problem['sol'] if p == 2.: problem = solvers.solve([f, functions.dummy(), h], x0=x0, solver=solver, **kwargs) return problem['sol'] else: return x0
def _solve(functions, x0, solver=None, atol=None, dtol=None, rtol=1e-3, xtol=None, maxit=200, verbosity='LOW'): r""" Solve an optimization problem whose objective function is the sum of some convex functions. This function minimizes the objective function :math:`f(x) = \sum\limits_{k=0}^{k=K} f_k(x)`, i.e. solves :math:`\operatorname{arg\,min}\limits_x f(x)` for :math:`x \in \mathbb{R}^{n \times N}` where :math:`n` is the dimensionality of the data and :math:`N` the number of independent problems. It returns a dictionary with the found solution and some information about the algorithm execution. Note ---- This code is taken from pyunlocbox. Our goal is to modify the function to also return intermediate solutions. Parameters ---------- functions : list of objects A list of convex functions to minimize. These are objects who must implement the :meth:`pyunlocbox.functions.func.eval` method. The :meth:`pyunlocbox.functions.func.grad` and / or :meth:`pyunlocbox.functions.func.prox` methods are required by some solvers. Note also that some solvers can only handle two convex functions while others may handle more. Please refer to the documentation of the considered solver. x0 : array_like Starting point of the algorithm, :math:`x_0 \in \mathbb{R}^{n \times N}`. Note that if you pass a numpy array it will be modified in place during execution to save memory. It will then contain the solution. Be careful to pass data of the type (int, float32, float64) you want your computations to use. solver : solver class instance, optional The solver algorithm. It is an object who must inherit from :class:`pyunlocbox.solvers.solver` and implement the :meth:`_pre`, :meth:`_algo` and :meth:`_post` methods. If no solver object are provided, a standard one will be chosen given the number of convex function objects and their implemented methods. atol : float, optional The absolute tolerance stopping criterion. The algorithm stops when :math:`f(x^t) < atol` where :math:`f(x^t)` is the objective function at iteration :math:`t`. Default is None. dtol : float, optional Stop when the objective function is stable enough, i.e. when :math:`\left|f(x^t) - f(x^{t-1})\right| < dtol`. Default is None. rtol : float, optional The relative tolerance stopping criterion. The algorithm stops when :math:`\left|\frac{ f(x^t) - f(x^{t-1}) }{ f(x^t) }\right| < rtol`. Default is :math:`10^{-3}`. xtol : float, optional Stop when the variable is stable enough, i.e. when :math:`\frac{\|x^t - x^{t-1}\|_2}{\sqrt{n N}} < xtol`. Note that additional memory will be used to store :math:`x^{t-1}`. Default is None. maxit : int, optional The maximum number of iterations. Default is 200. verbosity : {'NONE', 'LOW', 'HIGH', 'ALL'}, optional The log level : ``'NONE'`` for no log, ``'LOW'`` for resume at convergence, ``'HIGH'`` for info at all solving steps, ``'ALL'`` for all possible outputs, including at each steps of the proximal operators computation. Default is ``'LOW'``. Returns ------- sol : ndarray The problem solution. solver : str The used solver. crit : {'ATOL', 'DTOL', 'RTOL', 'XTOL', 'MAXIT'} The used stopping criterion. See above for definitions. niter : int The number of iterations. time : float The execution time in seconds. objective : ndarray The successive evaluations of the objective function at each iteration. backtrace : ndarray (N_iter + 1, len(sol)) past values of solution. """ if verbosity not in ['NONE', 'LOW', 'HIGH', 'ALL']: raise ValueError('Verbosity should be either NONE, LOW, HIGH or ALL.') # Add a second dummy convex function if only one function is provided. if len(functions) < 1: raise ValueError('At least 1 convex function should be provided.') elif len(functions) == 1: functions.append(dummy()) if verbosity in ['LOW', 'HIGH', 'ALL']: print('INFO: Dummy objective function added.') # Choose a solver if none provided. if not solver: if len(functions) == 2: fb0 = 'GRAD' in functions[0].cap(x0) and \ 'PROX' in functions[1].cap(x0) fb1 = 'GRAD' in functions[1].cap(x0) and \ 'PROX' in functions[0].cap(x0) dg0 = 'PROX' in functions[0].cap(x0) and \ 'PROX' in functions[1].cap(x0) if fb0 or fb1: solver = forward_backward() # Need one prox and 1 grad. elif dg0: solver = douglas_rachford() # Need two prox. else: raise ValueError('No suitable solver for the given functions.') elif len(functions) > 2: solver = generalized_forward_backward() if verbosity in ['LOW', 'HIGH', 'ALL']: name = solver.__class__.__name__ print('INFO: Selected solver: {}'.format(name)) # Set solver and functions verbosity. translation = {'ALL': 'HIGH', 'HIGH': 'HIGH', 'LOW': 'LOW', 'NONE': 'NONE'} solver.verbosity = translation[verbosity] translation = {'ALL': 'HIGH', 'HIGH': 'LOW', 'LOW': 'NONE', 'NONE': 'NONE'} functions_verbosity = [] for f in functions: functions_verbosity.append(f.verbosity) f.verbosity = translation[verbosity] tstart = time.time() crit = None niter = 0 objective = [[f.eval(x0) for f in functions]] rtol_only_zeros = True # Solver specific initialization. solver.pre(functions, x0) tape_buffer = np.zeros((1000, len(x0))) tape_buffer[0] = x0 while not crit: niter += 1 if xtol is not None: last_sol = np.array(solver.sol, copy=True) if verbosity in ['HIGH', 'ALL']: name = solver.__class__.__name__ print('Iteration {} of {}:'.format(niter, name)) # Solver iterative algorithm. solver.algo(objective, niter) tape_buffer[niter] = solver.sol objective.append([f.eval(solver.sol) for f in functions]) current = np.sum(objective[-1]) last = np.sum(objective[-2]) # Verify stopping criteria. if atol is not None and current < atol: crit = 'ATOL' if dtol is not None and np.abs(current - last) < dtol: crit = 'DTOL' if rtol is not None: div = current # Prevent division by 0. if div == 0: if verbosity in ['LOW', 'HIGH', 'ALL']: print('WARNING: (rtol) objective function is equal to 0 !') if last != 0: div = last else: div = 1.0 # Result will be zero anyway. else: rtol_only_zeros = False relative = np.abs((current - last) / div) if relative < rtol and not rtol_only_zeros: crit = 'RTOL' if xtol is not None: err = np.linalg.norm(solver.sol - last_sol) err /= np.sqrt(last_sol.size) if err < xtol: crit = 'XTOL' if maxit is not None and niter >= maxit: crit = 'MAXIT' if verbosity in ['HIGH', 'ALL']: print(' objective = {:.2e}'.format(current)) # Restore verbosity for functions. In case they are called outside solve(). for k, f in enumerate(functions): f.verbosity = functions_verbosity[k] if verbosity in ['LOW', 'HIGH', 'ALL']: print('Solution found after {} iterations:'.format(niter)) print(' objective function f(sol) = {:e}'.format(current)) print(' stopping criterion: {}'.format(crit)) # Returned dictionary. result = { 'sol': solver.sol, 'solver': solver.__class__.__name__, # algo for consistency ? 'crit': crit, 'niter': niter, 'time': time.time() - tstart, 'objective': objective } try: # Update dictionary for primal-dual solvers result['dual_sol'] = solver.dual_sol except AttributeError: pass # Solver specific post-processing (e.g. delete references). solver.post() result['backtrace'] = tape_buffer[:(niter + 1)] return result
def test_solve(self): """ Test some features of the solving function. """ # We have to set a seed here for the random draw if we are required # below to assert that the number of iterations of the solvers are # equal to some specific values. Otherwise, we get trivial errors when # x0 is a little farther away from y in a given draw. rs = np.random.RandomState(42) y = 5 - 10 * rs.uniform(size=(15, 4)) def x0(): return np.zeros(y.shape) nverb = {'verbosity': 'NONE'} # Function verbosity. f = functions.dummy() self.assertEqual(f.verbosity, 'NONE') f.verbosity = 'LOW' solvers.solve([f], x0(), **nverb) self.assertEqual(f.verbosity, 'LOW') # Input parameters. self.assertRaises(ValueError, solvers.solve, [f], x0(), verbosity='??') # Addition of dummy function. self.assertRaises(ValueError, solvers.solve, [], x0(), **nverb) solver = solvers.forward_backward() solvers.solve([f], x0(), solver, **nverb) # self.assertIsInstance(solver.f1, functions.dummy) # self.assertIsInstance(solver.f2, functions.dummy) # Automatic solver selection. f0 = functions.func() f0._eval = lambda x: 0 f0._grad = lambda x: x f1 = functions.func() f1._eval = lambda x: 0 f1._grad = lambda x: x f1._prox = lambda x, T: x f2 = functions.func() f2._eval = lambda x: 0 f2._prox = lambda x, T: x self.assertRaises(ValueError, solvers.solve, [f0, f0], x0(), **nverb) ret = solvers.solve([f0, f1], x0(), **nverb) self.assertEqual(ret['solver'], 'forward_backward') ret = solvers.solve([f1, f0], x0(), **nverb) self.assertEqual(ret['solver'], 'forward_backward') ret = solvers.solve([f1, f2], x0(), **nverb) self.assertEqual(ret['solver'], 'forward_backward') ret = solvers.solve([f2, f2], x0(), **nverb) self.assertEqual(ret['solver'], 'douglas_rachford') ret = solvers.solve([f1, f2, f0], x0(), **nverb) self.assertEqual(ret['solver'], 'generalized_forward_backward') # Stopping criteria. f = functions.norm_l2(y=y) tol = 1e-6 r = solvers.solve([f], x0(), None, tol, None, None, None, None, 'NONE') self.assertEqual(r['crit'], 'ATOL') self.assertLess(np.sum(r['objective'][-1]), tol) self.assertEqual(r['niter'], 9) tol = 1e-8 r = solvers.solve([f], x0(), None, None, tol, None, None, None, 'NONE') self.assertEqual(r['crit'], 'DTOL') err = np.abs(np.sum(r['objective'][-1]) - np.sum(r['objective'][-2])) self.assertLess(err, tol) self.assertEqual(r['niter'], 17) tol = .1 r = solvers.solve([f], x0(), None, None, None, tol, None, None, 'NONE') self.assertEqual(r['crit'], 'RTOL') err = np.abs(np.sum(r['objective'][-1]) - np.sum(r['objective'][-2])) err /= np.sum(r['objective'][-1]) self.assertLess(err, tol) self.assertEqual(r['niter'], 13) tol = 1e-4 r = solvers.solve([f], x0(), None, None, None, None, tol, None, 'NONE') self.assertEqual(r['crit'], 'XTOL') r2 = solvers.solve([f], x0(), maxit=r['niter'] - 1, **nverb) err = np.linalg.norm(r['sol'] - r2['sol']) / np.sqrt(x0().size) self.assertLess(err, tol) self.assertEqual(r['niter'], 14) nit = 15 r = solvers.solve([f], x0(), None, None, None, None, None, nit, 'NONE') self.assertEqual(r['crit'], 'MAXIT') self.assertEqual(r['niter'], nit) # Return values. f = functions.norm_l2(y=y) ret = solvers.solve([f], x0(), **nverb) self.assertEqual(len(ret), 6) self.assertIsInstance(ret['sol'], np.ndarray) self.assertIsInstance(ret['solver'], str) self.assertIsInstance(ret['crit'], str) self.assertIsInstance(ret['niter'], int) self.assertIsInstance(ret['time'], float) self.assertIsInstance(ret['objective'], list)
def solve(functions, x0, solver=None, atol=None, dtol=None, rtol=1e-3, xtol=None, maxit=200, verbosity='LOW'): r""" Solve an optimization problem whose objective function is the sum of some convex functions. This function minimizes the objective function :math:`f(x) = \sum\limits_{k=0}^{k=K} f_k(x)`, i.e. solves :math:`\operatorname{arg\,min}\limits_x f(x)` for :math:`x \in \mathbb{R}^{n \times N}` where :math:`n` is the dimensionality of the data and :math:`N` the number of independent problems. It returns a dictionary with the found solution and some informations about the algorithm execution. Parameters ---------- functions : list of objects A list of convex functions to minimize. These are objects who must implement the :meth:`pyunlocbox.functions.func.eval` method. The :meth:`pyunlocbox.functions.func.grad` and / or :meth:`pyunlocbox.functions.func.prox` methods are required by some solvers. Note also that some solvers can only handle two convex functions while others may handle more. Please refer to the documentation of the considered solver. x0 : array_like Starting point of the algorithm, :math:`x_0 \in \mathbb{R}^{n \times N}`. Note that if you pass a numpy array it will be modified in place during execution to save memory. It will then contain the solution. Be careful to pass data of the type (int, float32, float64) you want your computations to use. solver : solver class instance, optional The solver algorithm. It is an object who must inherit from :class:`pyunlocbox.solvers.solver` and implement the :meth:`_pre`, :meth:`_algo` and :meth:`_post` methods. If no solver object are provided, a standard one will be chosen given the number of convex function objects and their implemented methods. atol : float, optional The absolute tolerance stopping criterion. The algorithm stops when :math:`f(x^t) < atol` where :math:`f(x^t)` is the objective function at iteration :math:`t`. Default is None. dtol : float, optional Stop when the objective function is stable enough, i.e. when :math:`\left|f(x^t) - f(x^{t-1})\right| < dtol`. Default is None. rtol : float, optional The relative tolerance stopping criterion. The algorithm stops when :math:`\left|\frac{ f(x^t) - f(x^{t-1}) }{ f(x^t) }\right| < rtol`. Default is :math:`10^{-3}`. xtol : float, optional Stop when the variable is stable enough, i.e. when :math:`\frac{\|x^t - x^{t-1}\|_2}{\sqrt{n N}} < xtol`. Note that additional memory will be used to store :math:`x^{t-1}`. Default is None. maxit : int, optional The maximum number of iterations. Default is 200. verbosity : {'NONE', 'LOW', 'HIGH', 'ALL'}, optional The log level : ``'NONE'`` for no log, ``'LOW'`` for resume at convergence, ``'HIGH'`` for info at all solving steps, ``'ALL'`` for all possible outputs, including at each steps of the proximal operators computation. Default is ``'LOW'``. Returns ------- sol : ndarray The problem solution. solver : str The used solver. crit : {'ATOL', 'DTOL', 'RTOL', 'XTOL', 'MAXIT'} The used stopping criterion. See above for definitions. niter : int The number of iterations. time : float The execution time in seconds. objective : ndarray The successive evaluations of the objective function at each iteration. Examples -------- >>> import numpy as np >>> from pyunlocbox import functions, solvers Define a problem: >>> y = [4, 5, 6, 7] >>> f = functions.norm_l2(y=y) Solve it: >>> x0 = np.zeros(len(y)) >>> ret = solvers.solve([f], x0, atol=1e-2, verbosity='ALL') INFO: Dummy objective function added. INFO: Selected solver: forward_backward norm_l2 evaluation: 1.260000e+02 dummy evaluation: 0.000000e+00 INFO: Forward-backward method Iteration 1 of forward_backward: norm_l2 evaluation: 1.400000e+01 dummy evaluation: 0.000000e+00 objective = 1.40e+01 Iteration 2 of forward_backward: norm_l2 evaluation: 2.963739e-01 dummy evaluation: 0.000000e+00 objective = 2.96e-01 Iteration 3 of forward_backward: norm_l2 evaluation: 7.902529e-02 dummy evaluation: 0.000000e+00 objective = 7.90e-02 Iteration 4 of forward_backward: norm_l2 evaluation: 5.752265e-02 dummy evaluation: 0.000000e+00 objective = 5.75e-02 Iteration 5 of forward_backward: norm_l2 evaluation: 5.142032e-03 dummy evaluation: 0.000000e+00 objective = 5.14e-03 Solution found after 5 iterations: objective function f(sol) = 5.142032e-03 stopping criterion: ATOL Verify the stopping criterion (should be smaller than atol=1e-2): >>> np.linalg.norm(ret['sol'] - y)**2 # doctest:+ELLIPSIS 0.00514203... Show the solution (should be close to y w.r.t. the L2-norm measure): >>> ret['sol'] array([4.02555301, 5.03194126, 6.03832952, 7.04471777]) Show the used solver: >>> ret['solver'] 'forward_backward' Show some information about the convergence: >>> ret['crit'] 'ATOL' >>> ret['niter'] 5 >>> ret['time'] # doctest:+SKIP 0.0012578964233398438 >>> ret['objective'] # doctest:+NORMALIZE_WHITESPACE,+ELLIPSIS [[126.0, 0], [13.99999999..., 0], [0.29637392..., 0], [0.07902528..., 0], [0.05752265..., 0], [0.00514203..., 0]] """ if verbosity not in ['NONE', 'LOW', 'HIGH', 'ALL']: raise ValueError('Verbosity should be either NONE, LOW, HIGH or ALL.') # Add a second dummy convex function if only one function is provided. if len(functions) < 1: raise ValueError('At least 1 convex function should be provided.') elif len(functions) == 1: functions.append(dummy()) if verbosity in ['LOW', 'HIGH', 'ALL']: print('INFO: Dummy objective function added.') # Choose a solver if none provided. if not solver: if len(functions) == 2: fb0 = 'GRAD' in functions[0].cap(x0) and \ 'PROX' in functions[1].cap(x0) fb1 = 'GRAD' in functions[1].cap(x0) and \ 'PROX' in functions[0].cap(x0) dg0 = 'PROX' in functions[0].cap(x0) and \ 'PROX' in functions[1].cap(x0) if fb0 or fb1: solver = forward_backward() # Need one prox and 1 grad. elif dg0: solver = douglas_rachford() # Need two prox. else: raise ValueError('No suitable solver for the given functions.') elif len(functions) > 2: solver = generalized_forward_backward() if verbosity in ['LOW', 'HIGH', 'ALL']: name = solver.__class__.__name__ print('INFO: Selected solver: {}'.format(name)) # Set solver and functions verbosity. translation = {'ALL': 'HIGH', 'HIGH': 'HIGH', 'LOW': 'LOW', 'NONE': 'NONE'} solver.verbosity = translation[verbosity] translation = {'ALL': 'HIGH', 'HIGH': 'LOW', 'LOW': 'NONE', 'NONE': 'NONE'} functions_verbosity = [] for f in functions: functions_verbosity.append(f.verbosity) f.verbosity = translation[verbosity] tstart = time.time() crit = None niter = 0 objective = [[f.eval(x0) for f in functions]] rtol_only_zeros = True # Solver specific initialization. solver.pre(functions, x0) while not crit: niter += 1 if xtol is not None: last_sol = np.array(solver.sol, copy=True) if verbosity in ['HIGH', 'ALL']: name = solver.__class__.__name__ print('Iteration {} of {}:'.format(niter, name)) # Solver iterative algorithm. solver.algo(objective, niter) objective.append([f.eval(solver.sol) for f in functions]) current = np.sum(objective[-1]) last = np.sum(objective[-2]) # Verify stopping criteria. if atol is not None and current < atol: crit = 'ATOL' if dtol is not None and np.abs(current - last) < dtol: crit = 'DTOL' if rtol is not None: div = current # Prevent division by 0. if div == 0: if verbosity in ['LOW', 'HIGH', 'ALL']: print('WARNING: (rtol) objective function is equal to 0 !') if last != 0: div = last else: div = 1.0 # Result will be zero anyway. else: rtol_only_zeros = False relative = np.abs((current - last) / div) if relative < rtol and not rtol_only_zeros: crit = 'RTOL' if xtol is not None: err = np.linalg.norm(solver.sol - last_sol) err /= np.sqrt(last_sol.size) if err < xtol: crit = 'XTOL' if maxit is not None and niter >= maxit: crit = 'MAXIT' if verbosity in ['HIGH', 'ALL']: print(' objective = {:.2e}'.format(current)) # Restore verbosity for functions. In case they are called outside solve(). for k, f in enumerate(functions): f.verbosity = functions_verbosity[k] if verbosity in ['LOW', 'HIGH', 'ALL']: print('Solution found after {} iterations:'.format(niter)) print(' objective function f(sol) = {:e}'.format(current)) print(' stopping criterion: {}'.format(crit)) # Returned dictionary. result = {'sol': solver.sol, 'solver': solver.__class__.__name__, # algo for consistency ? 'crit': crit, 'niter': niter, 'time': time.time() - tstart, 'objective': objective} try: # Update dictionary for primal-dual solvers result['dual_sol'] = solver.dual_sol except AttributeError: pass # Solver specific post-processing (e.g. delete references). solver.post() return result
def test_mlfbf(self): """ Test the MLFBF solver with arbitrarily selected functions. """ x = [1., 1., 1.] L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]]) max_step = 1 / (1 + np.linalg.norm(L, 2)) solver = solvers.mlfbf(L=L, step=max_step / 2.) params = {'solver': solver, 'verbosity': 'NONE'} def x0(): return np.zeros(len(x)) # L2-norm prox and dummy prox. f = functions.dummy() f._prox = lambda x, T: np.maximum(np.zeros(len(x)), x) g = functions.norm_l2(lambda_=0.5) h = functions.norm_l2(y=np.array([294, 390, 361]), lambda_=0.5) ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params) nptest.assert_allclose(ret['sol'], x, rtol=1e-5) # Same test, but with callable L solver = solvers.mlfbf(L=lambda x: np.dot(L, x), Lt=lambda y: np.dot(L.T, y), d0=np.dot(L, x0()), step=max_step / 2.) ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params) nptest.assert_allclose(ret['sol'], x, rtol=1e-5) # Sanity check self.assertRaises(ValueError, solver.pre, [f, g], x0()) # Make a second test where the solution is calculated by hand n = 10 y = np.random.rand(n) * 2 z = np.random.rand(n) c = 1 delta = (y - z - c)**2 + 4 * (1 + y * z - z * c) sol = 0.5 * ((y - z - c) + np.sqrt(delta)) class mlog(functions.func): def __init__(self, z): super().__init__() self.z = z def _eval(self, x): return -np.sum(np.log(x + self.z)) def _prox(self, x, T): delta = (x - self.z)**2 + 4 * (T + x * self.z) sol = 0.5 * (x - self.z + np.sqrt(delta)) return sol f = functions.norm_l1(lambda_=c) g = mlog(z=z) h = functions.norm_l2(lambda_=0.5, y=y) mu = 1 + 1 step = 1 / mu / 2 solver = solvers.mlfbf(step=step) ret = solvers.solve([f, g, h], y.copy(), solver, maxit=200, rtol=0, verbosity="NONE") nptest.assert_allclose(ret["sol"], sol, atol=1e-10) # Make a final test where the function g can not be evaluate # on the primal variables y = np.random.rand(3) y_2 = L.dot(y) L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]]) x0 = np.zeros(len(y)) f = functions.norm_l1(y=y) g = functions.norm_l2(lambda_=0.5, y=y_2) h = functions.norm_l2(y=y, lambda_=0.5) max_step = 1 / (1 + np.linalg.norm(L, 2)) solver = solvers.mlfbf(L=L, step=max_step / 2.) ret = solvers.solve([f, g, h], x0, solver, maxit=1000, rtol=0) np.testing.assert_allclose(ret["sol"], y)
def solve(functions, x0, solver=None, atol=None, dtol=None, rtol=1e-3, xtol=None, maxit=200, verbosity='LOW'): r""" Solve an optimization problem whose objective function is the sum of some convex functions. This function minimizes the objective function :math:`f(x) = \sum\limits_{k=0}^{k=K} f_k(x)`, i.e. solves :math:`\operatorname{arg\,min}\limits_x f(x)` for :math:`x \in \mathbb{R}^{n \times N}` where :math:`n` is the dimensionality of the data and :math:`N` the number of independent problems. It returns a dictionary with the found solution and some informations about the algorithm execution. Parameters ---------- functions : list of objects A list of convex functions to minimize. These are objects who must implement the :meth:`pyunlocbox.functions.func.eval` method. The :meth:`pyunlocbox.functions.func.grad` and / or :meth:`pyunlocbox.functions.func.prox` methods are required by some solvers. Note also that some solvers can only handle two convex functions while others may handle more. Please refer to the documentation of the considered solver. x0 : array_like Starting point of the algorithm, :math:`x_0 \in \mathbb{R}^{n \times N}`. Note that if you pass a numpy array it will be modified in place during execution to save memory. It will then contain the solution. Be careful to pass data of the type (int, float32, float64) you want your computations to use. solver : solver class instance, optional The solver algorithm. It is an object who must inherit from :class:`pyunlocbox.solvers.solver` and implement the :meth:`_pre`, :meth:`_algo` and :meth:`_post` methods. If no solver object are provided, a standard one will be chosen given the number of convex function objects and their implemented methods. atol : float, optional The absolute tolerance stopping criterion. The algorithm stops when :math:`f(x^t) < atol` where :math:`f(x^t)` is the objective function at iteration :math:`t`. Default is None. dtol : float, optional Stop when the objective function is stable enough, i.e. when :math:`\left|f(x^t) - f(x^{t-1})\right| < dtol`. Default is None. rtol : float, optional The relative tolerance stopping criterion. The algorithm stops when :math:`\left|\frac{ f(x^t) - f(x^{t-1}) }{ f(x^t) }\right| < rtol`. Default is :math:`10^{-3}`. xtol : float, optional Stop when the variable is stable enough, i.e. when :math:`\frac{\|x^t - x^{t-1}\|_2}{\sqrt{n N}} < xtol`. Note that additional memory will be used to store :math:`x^{t-1}`. Default is None. maxit : int, optional The maximum number of iterations. Default is 200. verbosity : {'NONE', 'LOW', 'HIGH', 'ALL'}, optional The log level : ``'NONE'`` for no log, ``'LOW'`` for resume at convergence, ``'HIGH'`` for info at all solving steps, ``'ALL'`` for all possible outputs, including at each steps of the proximal operators computation. Default is ``'LOW'``. Returns ------- sol : ndarray The problem solution. solver : str The used solver. crit : {'ATOL', 'DTOL', 'RTOL', 'XTOL', 'MAXIT'} The used stopping criterion. See above for definitions. niter : int The number of iterations. time : float The execution time in seconds. objective : ndarray The successive evaluations of the objective function at each iteration. Examples -------- >>> import numpy as np >>> from pyunlocbox import functions, solvers Define a problem: >>> y = [4, 5, 6, 7] >>> f = functions.norm_l2(y=y) Solve it: >>> x0 = np.zeros(len(y)) >>> ret = solvers.solve([f], x0, atol=1e-2, verbosity='ALL') INFO: Dummy objective function added. INFO: Selected solver: forward_backward norm_l2 evaluation: 1.260000e+02 dummy evaluation: 0.000000e+00 INFO: Forward-backward method Iteration 1 of forward_backward: norm_l2 evaluation: 1.400000e+01 dummy evaluation: 0.000000e+00 objective = 1.40e+01 Iteration 2 of forward_backward: norm_l2 evaluation: 2.963739e-01 dummy evaluation: 0.000000e+00 objective = 2.96e-01 Iteration 3 of forward_backward: norm_l2 evaluation: 7.902529e-02 dummy evaluation: 0.000000e+00 objective = 7.90e-02 Iteration 4 of forward_backward: norm_l2 evaluation: 5.752265e-02 dummy evaluation: 0.000000e+00 objective = 5.75e-02 Iteration 5 of forward_backward: norm_l2 evaluation: 5.142032e-03 dummy evaluation: 0.000000e+00 objective = 5.14e-03 Solution found after 5 iterations: objective function f(sol) = 5.142032e-03 stopping criterion: ATOL Verify the stopping criterion (should be smaller than atol=1e-2): >>> np.linalg.norm(ret['sol'] - y)**2 # doctest:+ELLIPSIS 0.00514203... Show the solution (should be close to y w.r.t. the L2-norm measure): >>> ret['sol'] array([4.02555301, 5.03194126, 6.03832952, 7.04471777]) Show the used solver: >>> ret['solver'] 'forward_backward' Show some information about the convergence: >>> ret['crit'] 'ATOL' >>> ret['niter'] 5 >>> ret['time'] # doctest:+SKIP 0.0012578964233398438 >>> ret['objective'] # doctest:+NORMALIZE_WHITESPACE,+ELLIPSIS [[126.0, 0], [13.99999999..., 0], [0.29637392..., 0], [0.07902528..., 0], [0.05752265..., 0], [0.00514203..., 0]] """ if verbosity not in ['NONE', 'LOW', 'HIGH', 'ALL']: raise ValueError('Verbosity should be either NONE, LOW, HIGH or ALL.') # Add a second dummy convex function if only one function is provided. if len(functions) < 1: raise ValueError('At least 1 convex function should be provided.') elif len(functions) == 1: functions.append(dummy()) if verbosity in ['LOW', 'HIGH', 'ALL']: print('INFO: Dummy objective function added.') # Choose a solver if none provided. if not solver: if len(functions) == 2: fb0 = 'GRAD' in functions[0].cap(x0) and \ 'PROX' in functions[1].cap(x0) fb1 = 'GRAD' in functions[1].cap(x0) and \ 'PROX' in functions[0].cap(x0) dg0 = 'PROX' in functions[0].cap(x0) and \ 'PROX' in functions[1].cap(x0) if fb0 or fb1: solver = forward_backward() # Need one prox and 1 grad. elif dg0: solver = douglas_rachford() # Need two prox. else: raise ValueError('No suitable solver for the given functions.') elif len(functions) > 2: solver = generalized_forward_backward() if verbosity in ['LOW', 'HIGH', 'ALL']: name = solver.__class__.__name__ print('INFO: Selected solver: {}'.format(name)) # Set solver and functions verbosity. translation = {'ALL': 'HIGH', 'HIGH': 'HIGH', 'LOW': 'LOW', 'NONE': 'NONE'} solver.verbosity = translation[verbosity] translation = {'ALL': 'HIGH', 'HIGH': 'LOW', 'LOW': 'NONE', 'NONE': 'NONE'} functions_verbosity = [] for f in functions: functions_verbosity.append(f.verbosity) f.verbosity = translation[verbosity] tstart = time.time() crit = None niter = 0 objective = [[f.eval(x0) for f in functions]] rtol_only_zeros = True # Solver specific initialization. solver.pre(functions, x0) while not crit: niter += 1 if xtol is not None: last_sol = np.array(solver.sol, copy=True) if verbosity in ['HIGH', 'ALL']: name = solver.__class__.__name__ print('Iteration {} of {}:'.format(niter, name)) # Solver iterative algorithm. solver.algo(objective, niter) objective.append([f.eval(solver.sol) for f in functions]) current = np.sum(objective[-1]) last = np.sum(objective[-2]) # Verify stopping criteria. if atol is not None and current < atol: crit = 'ATOL' if dtol is not None and np.abs(current - last) < dtol: crit = 'DTOL' if rtol is not None: div = current # Prevent division by 0. if div == 0: if verbosity in ['LOW', 'HIGH', 'ALL']: print('WARNING: (rtol) objective function is equal to 0 !') if last != 0: div = last else: div = 1.0 # Result will be zero anyway. else: rtol_only_zeros = False relative = np.abs((current - last) / div) if relative < rtol and not rtol_only_zeros: crit = 'RTOL' if xtol is not None: err = np.linalg.norm(solver.sol - last_sol) err /= np.sqrt(last_sol.size) if err < xtol: crit = 'XTOL' if maxit is not None and niter >= maxit: crit = 'MAXIT' if verbosity in ['HIGH', 'ALL']: print(' objective = {:.2e}'.format(current)) # Restore verbosity for functions. In case they are called outside solve(). for k, f in enumerate(functions): f.verbosity = functions_verbosity[k] if verbosity in ['LOW', 'HIGH', 'ALL']: print('Solution found after {} iterations:'.format(niter)) print(' objective function f(sol) = {:e}'.format(current)) print(' stopping criterion: {}'.format(crit)) # Returned dictionary. result = { 'sol': solver.sol, 'solver': solver.__class__.__name__, # algo for consistency ? 'crit': crit, 'niter': niter, 'time': time.time() - tstart, 'objective': objective } try: # Update dictionary for primal-dual solvers result['dual_sol'] = solver.dual_sol except AttributeError: pass # Solver specific post-processing (e.g. delete references). solver.post() return result
def test_forward_backward_fista(self): """ Test forward-backward splitting solver with fista acceleration, solving problems with L1-norm, L2-norm, and dummy functions. """ y = [4., 5., 6., 7.] solver = solvers.forward_backward(accel=acceleration.fista()) param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'} # L2-norm prox and dummy gradient. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 60) # Dummy prox and L2-norm gradient. f1 = functions.dummy() f2 = functions.norm_l2(y=y, lambda_=0.6) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 84) # L2-norm prox and L2-norm gradient. f1 = functions.norm_l2(y=y) f2 = functions.norm_l2(y=y) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y, rtol=1e-2) self.assertEqual(ret['crit'], 'MAXIT') self.assertEqual(ret['niter'], 200) # L1-norm prox and dummy gradient. f1 = functions.norm_l1(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 6) # Dummy prox and L1-norm gradient. As L1-norm possesses no gradient, # the algorithm exchanges the functions : exact same solution. f1 = functions.dummy() f2 = functions.norm_l1(y=y) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 6) # L1-norm prox and L1-norm gradient. L1-norm possesses no gradient. f1 = functions.norm_l1(y=y) f2 = functions.norm_l1(y=y) self.assertRaises(ValueError, solvers.solve, [f1, f2], np.zeros(len(y)), **param) # L1-norm prox and L2-norm gradient. f1 = functions.norm_l1(y=y, lambda_=1.0) f2 = functions.norm_l2(y=y, lambda_=0.8) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 10)