def test_projection_based(self):
"""
Test the projection-based solver with arbitrarily selected functions.
"""
x = [0, 0, 0]
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
solver = solvers.projection_based(L=L, step=1.)
params = {'solver': solver, 'verbosity': 'NONE'}
# L1-norm prox and dummy prox.
f = functions.norm_l1(y=np.array([294, 390, 361]))
g = functions.norm_l1()
ret = solvers.solve([f, g],
np.array([500, 1000, -400]),
maxit=1000,
rtol=None,
xtol=0.1,
**params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Sanity checks
def x0():
return np.zeros(len(x))
self.assertRaises(ValueError, solver.pre, [f], x0())
solver.lambda_ = 3.
self.assertRaises(ValueError, solver.pre, [f, g], x0())
solver.lambda_ = -3.
self.assertRaises(ValueError, solver.pre, [f, g], x0())
def test_forward_backward_fista_backtracking(self):
"""
Test forward-backward splitting solver with fista acceleration and
backtracking, solving problems with L1-norm, L2-norm, and dummy
functions.
"""
y = [4., 5., 6., 7.]
accel = acceleration.fista_backtracking()
solver = solvers.forward_backward(accel=accel)
param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 60)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 3)
def test_backtracking(self):
"""
Test forward-backward splitting solver with backtracking, solving
problems with L1-norm, L2-norm, and dummy functions.
"""
# Test constructor sanity
a = acceleration.backtracking()
self.assertRaises(ValueError, a.__init__, 2.)
self.assertRaises(ValueError, a.__init__, -2.)
y = [4., 5., 6., 7.]
accel = acceleration.backtracking()
step = 10 # Make sure backtracking is called
solver = solvers.forward_backward(accel=accel, step=step)
param = {'solver': solver, 'atol': 1e-32, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'ATOL')
self.assertEqual(ret['niter'], 13)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'ATOL')
self.assertLessEqual(ret['niter'], 4) # win64 takes one iteration
def test_forward_backward(self):
"""
Test forward-backward splitting algorithm without acceleration, and
with L1-norm, L2-norm, and dummy functions.
"""
y = [4., 5., 6., 7.]
solver = solvers.forward_backward(accel=acceleration.dummy())
param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 35)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 4)
# Sanity check
f3 = functions.dummy()
x0 = np.zeros((4, ))
self.assertRaises(ValueError, solver.pre, [f1, f2, f3], x0)
def test_douglas_rachford(self):
"""
Test douglas-rachford solver with L1-norm, L2-norm and dummy functions.
"""
y = [4, 5, 6, 7]
solver = solvers.douglas_rachford()
param = {'solver': solver, 'verbosity': 'NONE'}
# L2-norm prox and dummy prox.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 35)
# L2-norm prox and L1-norm prox.
f1 = functions.norm_l2(y=y)
f2 = functions.norm_l1(y=y)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 4)
# Sanity checks
x0 = np.zeros((4, ))
solver.lambda_ = 2.
self.assertRaises(ValueError, solver.pre, [f1, f2], x0)
solver.lambda_ = -2.
self.assertRaises(ValueError, solver.pre, [f1, f2], x0)
self.assertRaises(ValueError, solver.pre, [f1, f2, f1], x0)
def test_solver_comparison(self):
"""
Test that all solvers return the same and correct solution.
"""
# Convex functions.
y = [1, 0, 0.1, 8, -6.5, 0.2, 0.004, 0.01]
sol = [0.75, 0, 0, 7.75, -6.25, 0, 0, 0]
w1, w2 = .8, .4
f1 = functions.norm_l2(y=y, lambda_=w1 / 2.) # Smooth.
f2 = functions.norm_l1(lambda_=w2 / 2.) # Non-smooth.
# Solvers.
L = w1 # Lipschitz continuous gradient.
step = 1. / L
lambda_ = 0.5
params = {'step': step, 'lambda_': lambda_}
slvs = []
slvs.append(solvers.forward_backward(accel=acceleration.dummy(),
step=step))
slvs.append(solvers.douglas_rachford(**params))
slvs.append(solvers.generalized_forward_backward(**params))
# Compare solutions.
params = {'rtol': 1e-14, 'verbosity': 'NONE', 'maxit': 1e4}
niters = [2, 61, 26]
for solver, niter in zip(slvs, niters):
x0 = np.zeros(len(y))
ret = solvers.solve([f1, f2], x0, solver, **params)
nptest.assert_allclose(ret['sol'], sol)
self.assertEqual(ret['niter'], niter)
self.assertIs(ret['sol'], x0) # The initial value was modified.
def test_backtracking(self):
"""
Test forward-backward splitting solver with backtracking, solving
problems with L1-norm, L2-norm, and dummy functions.
"""
# Test constructor sanity
a = acceleration.backtracking()
self.assertRaises(ValueError, a.__init__, 2.)
self.assertRaises(ValueError, a.__init__, -2.)
y = [4., 5., 6., 7.]
accel = acceleration.backtracking()
step = 10 # Make sure backtracking is called
solver = solvers.forward_backward(accel=accel, step=step)
param = {'solver': solver, 'atol': 1e-32, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'ATOL')
self.assertEqual(ret['niter'], 13)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'ATOL')
self.assertEqual(ret['niter'], 4)
def test_forward_backward_fista_backtracking(self):
"""
Test forward-backward splitting solver with fista acceleration and
backtracking, solving problems with L1-norm, L2-norm, and dummy
functions.
"""
y = [4., 5., 6., 7.]
accel = acceleration.fista_backtracking()
solver = solvers.forward_backward(accel=accel)
param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 60)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 3)
def test_douglas_rachford(self):
"""
Test douglas-rachford solver with L1-norm, L2-norm and dummy functions.
"""
y = [4, 5, 6, 7]
solver = solvers.douglas_rachford()
param = {'solver': solver, 'verbosity': 'NONE'}
# L2-norm prox and dummy prox.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 35)
# L2-norm prox and L1-norm prox.
f1 = functions.norm_l2(y=y)
f2 = functions.norm_l1(y=y)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 4)
# Sanity checks
x0 = np.zeros((4,))
solver.lambda_ = 2.
self.assertRaises(ValueError, solver.pre, [f1, f2], x0)
solver.lambda_ = -2.
self.assertRaises(ValueError, solver.pre, [f1, f2], x0)
self.assertRaises(ValueError, solver.pre, [f1, f2, f1], x0)
def test_forward_backward(self):
"""
Test forward-backward splitting algorithm without acceleration, and
with L1-norm, L2-norm, and dummy functions.
"""
y = [4., 5., 6., 7.]
solver = solvers.forward_backward(accel=acceleration.dummy())
param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 35)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 4)
# Sanity check
f3 = functions.dummy()
x0 = np.zeros((4,))
self.assertRaises(ValueError, solver.pre, [f1, f2, f3], x0)
def test_solver_comparison(self):
"""
Test that all solvers return the same and correct solution.
"""
# Convex functions.
y = [1, 0, 0.1, 8, -6.5, 0.2, 0.004, 0.01]
sol = [0.75, 0, 0, 7.75, -6.25, 0, 0, 0]
w1, w2 = .8, .4
f1 = functions.norm_l2(y=y, lambda_=w1 / 2.) # Smooth.
f2 = functions.norm_l1(lambda_=w2 / 2.) # Non-smooth.
# Solvers.
L = w1 # Lipschitz continuous gradient.
step = 1. / L
lambda_ = 0.5
params = {'step': step, 'lambda_': lambda_}
slvs = []
slvs.append(
solvers.forward_backward(accel=acceleration.dummy(), step=step))
slvs.append(solvers.douglas_rachford(**params))
slvs.append(solvers.generalized_forward_backward(**params))
# Compare solutions.
params = {'rtol': 1e-14, 'verbosity': 'NONE', 'maxit': 1e4}
niters = [2, 61, 26]
for solver, niter in zip(slvs, niters):
x0 = np.zeros(len(y))
ret = solvers.solve([f1, f2], x0, solver, **params)
nptest.assert_allclose(ret['sol'], sol)
self.assertEqual(ret['niter'], niter)
self.assertIs(ret['sol'], x0) # The initial value was modified.
def test_acceleration_comparison(self):
"""
Test that all solvers return the same and correct solution.
"""
# Convex functions.
y = [1, 0, 0.1, 8, -6.5, 0.2, 0.004, 0.01]
sol = [0.75, 0, 0, 7.75, -6.25, 0, 0, 0]
w1, w2 = .8, .4
f1 = functions.norm_l2(y=y, lambda_=w1 / 2.) # Smooth.
f2 = functions.norm_l1(lambda_=w2 / 2.) # Non-smooth.
# Solvers.
L = w1 # Lipschitz continuous gradient.
step = 1. / L
slvs = []
slvs.append(solvers.forward_backward(accel=acceleration.dummy(),
step=step))
slvs.append(solvers.forward_backward(accel=acceleration.fista(),
step=step))
slvs.append(solvers.forward_backward(
accel=acceleration.fista_backtracking(eta=.999), step=step))
# Compare solutions.
params = {'rtol': 1e-14, 'verbosity': 'NONE', 'maxit': 1e4}
niters = [2, 2, 6]
for solver, niter in zip(slvs, niters):
x0 = np.zeros(len(y))
ret = solvers.solve([f1, f2], x0, solver, **params)
nptest.assert_allclose(ret['sol'], sol)
self.assertEqual(ret['niter'], niter)
def test_acceleration_comparison(self):
"""
Test that all solvers return the same and correct solution.
"""
# Convex functions.
y = [1, 0, 0.1, 8, -6.5, 0.2, 0.004, 0.01]
sol = [0.75, 0, 0, 7.75, -6.25, 0, 0, 0]
w1, w2 = .8, .4
f1 = functions.norm_l2(y=y, lambda_=w1 / 2.) # Smooth.
f2 = functions.norm_l1(lambda_=w2 / 2.) # Non-smooth.
# Solvers.
L = w1 # Lipschitz continuous gradient.
step = 1. / L
slvs = []
slvs.append(
solvers.forward_backward(accel=acceleration.dummy(), step=step))
slvs.append(
solvers.forward_backward(accel=acceleration.fista(), step=step))
slvs.append(
solvers.forward_backward(
accel=acceleration.fista_backtracking(eta=.999), step=step))
# Compare solutions.
params = {'rtol': 1e-14, 'verbosity': 'NONE', 'maxit': 1e4}
niters = [2, 2, 6]
for solver, niter in zip(slvs, niters):
x0 = np.zeros(len(y))
ret = solvers.solve([f1, f2], x0, solver, **params)
nptest.assert_allclose(ret['sol'], sol)
self.assertEqual(ret['niter'], niter)
def optGFB(D, GK0, triangles, fxy, E, maxit, x1, step, tau1, tau2, tau3, flag):
####
N = len(E)
f2 = functions.func()
f2._eval = lambda x: 0
f2._prox = lambda x, T: np.clip(x, 0, 0.7) #
###
T1 = Nodeneighbor(triangles, N)
g = lambda x: Ggradient(x, T1)
f3 = functions.norm_l1(A=g, At=None, dim=1, y=np.zeros(N),
lambda_=tau2) #['EVAL', 'PROX']
######
yy2 = np.ravel(fxy)
####
N_cov = np.zeros((2 * N, 2 * N))
for j in np.arange(0, 2 * N, 2):
N_cov[j, j] = 3
N_cov[j + 1, j + 1] = 1
gamma = GK0 @ N_cov @ GK0.T #
gamma += np.eye(gamma.shape[1]) * 1e-5
gamma_inv = np.linalg.inv(gamma)
f8 = functions.func(lambda_=tau1)
f8._eval = lambda x: (yy2 - D @ x).T @ gamma_inv @ (yy2 - D @ x) * 1e+7
f8._grad = lambda x: -D.T @ gamma_inv @ (yy2 - D @ x) * 1e+7
#######
#step = 0.08#0.5 /tau1# (np.linalg.norm(func(x0),2)**2/np.linalg.norm(x0,2)**2) #0.5/tau#2e3/scale
solver2 = solvers.generalized_forward_backward(
step=step * 0.2
) #generalized_forward_backward(step=step*0.1)#step*0.1)douglas_rachford
# without f3 -->singular matrix
ret2 = solvers.solve([f8, f3, f2], x1, solver2, rtol=1e-15, maxit=maxit)
objective = np.array(ret2['objective'])
sol = ret2['sol']
import matplotlib.pyplot as plt
if flag == 1:
_ = plt.figure(figsize=(10, 4))
_ = plt.subplot(121)
_ = plt.plot(E, 'o', label='Original E') #-np.ones(N)*0.1
_ = plt.plot(ret2['sol'], 'xr', label='Reconstructed E')
_ = plt.grid(True)
_ = plt.title('Achieved reconstruction')
_ = plt.legend(numpoints=1)
_ = plt.xlabel('Signal dimension number')
_ = plt.ylabel('Signal value')
_ = plt.subplot(122)
_ = plt.semilogy(objective[:, 0], '-.', label='l2-norm') #
_ = plt.semilogy(np.sum(objective, axis=1), label='Global objective')
_ = plt.grid(True)
_ = plt.title('Convergence')
_ = plt.legend(numpoints=1)
_ = plt.xlabel('Iteration number')
_ = plt.ylabel('Objective function value')
_ = plt.show()
return sol
def test_norm_l1(self):
"""
Test the norm_l1 derived class.
We test the two methods : eval and prox.
First with default class properties, then custom ones.
"""
f = functions.norm_l1(lambda_=3)
self.assertEqual(f.eval([10, 0]), 30)
self.assertEqual(f.eval(np.array([-10, 0])), 30)
self.assertEqual(f.eval([-3, 4]), 21)
nptest.assert_array_equal(f.prox(np.array([[1, -4], [5, -2]]), 1),
[[0, -1], [2, 0]])
f = functions.norm_l1(tight=False)
x = np.ones((4, ))
T = 0.5
self.assertRaises(NotImplementedError, f.prox, x, T)
def test_primal_dual_solver_comparison(self):
"""
Test that all primal-dual solvers return the same and correct solution.
I had to create this separate function because the primal-dual solvers
were too slow for the problem above.
"""
# Convex functions.
y = np.random.randn(3)
L = np.random.randn(4, 3)
sol = y
y2 = L.dot(y)
f1 = functions.norm_l1(y=y)
f2 = functions.norm_l2(y=y2)
f3 = functions.dummy()
# Solvers.
step = 0.5 / (1 + np.linalg.norm(L, 2))
slvs = []
slvs.append(solvers.mlfbf(step=step, L=L))
slvs.append(solvers.projection_based(step=step, L=L))
# Compare solutions.
niter = 1000
params = {'rtol': 0, 'verbosity': 'NONE', 'maxit': niter}
for solver in slvs:
x0 = np.zeros(len(y))
if type(solver) is solvers.mlfbf:
ret = solvers.solve([f1, f2, f3], x0, solver, **params)
else:
ret = solvers.solve([f1, f2], x0, solver, **params)
nptest.assert_allclose(ret['sol'], sol)
self.assertEqual(ret['niter'], niter)
# The initial value was not modified.
nptest.assert_array_equal(x0, np.zeros(len(y)))
if type(solver) is solvers.mlfbf:
ret = solvers.solve([f1, f2, f3],
x0,
solver,
inplace=True,
**params)
else:
ret = solvers.solve([f1, f2],
x0,
solver,
inplace=True,
**params)
# The initial value was modified.
self.assertIs(ret['sol'], x0)
nptest.assert_allclose(ret['sol'], sol)
def test_norm_l1(self):
"""
Test the norm_l1 derived class.
We test the two methods : eval and prox.
First with default class properties, then custom ones.
"""
f = functions.norm_l1(lambda_=3)
self.assertEqual(f.eval([10, 0]), 30)
self.assertEqual(f.eval(np.array([-10, 0])), 30)
self.assertEqual(f.eval([-3, 4]), 21)
nptest.assert_array_equal(f.prox(np.array([[1, -4], [5, -2]]), 1),
[[0, -1], [2, 0]])
f = functions.norm_l1(tight=False)
x = np.ones((4, ))
T = 0.5
self.assertRaises(NotImplementedError, f.prox, x, T)
# Let us test the weights
f2 = functions.norm_l1(lambda_=4, w=2)
self.assertEqual(f2.eval([10, 1]), 88)
self.assertEqual(f2.eval([5]), 40)
self.assertEqual(f2.eval([0, -2, 1]), 24)
self.assertEqual(f2.eval(np.array([-10, 0])), 80)
self.assertEqual(f2.eval([-3, 4]), 56)
nptest.assert_array_equal(f2.prox(np.array([[7, -9], [10, -2]]), 1),
[[0, -1], [2, 0]])
# Let us multiple weights
f3 = functions.norm_l1(lambda_=4, w=[1, 2])
self.assertEqual(f3.eval([10, 1]), 48)
nptest.assert_array_equal(f3.prox(np.array([[7, -9], [10, -2]]), 1),
[[3, -1], [6, 0]])
# Let us test y
f3 = functions.norm_l1(lambda_=2, y=[1, 2])
self.assertEqual(f3.eval([10, 1]), 20)
nptest.assert_array_equal(f3.prox(np.array([[2, 5], [-1, -2]]), 1),
[[1, 3], [1, 0]])
def test_generalized_forward_backward(self):
"""
Test the generalized forward-backward algorithm.
"""
y = [4, 5, 6, 7]
L = 4 # Gradient of the smooth function is Lipschitz continuous.
solver = solvers.generalized_forward_backward(step=.9 / L, lambda_=.8)
params = {'solver': solver, 'verbosity': 'NONE'}
# Functions.
f1 = functions.norm_l1(y=y, lambda_=.7) # Non-smooth.
f2 = functions.norm_l2(y=y, lambda_=L / 2.) # Smooth.
# Solve with 1 smooth and 1 non-smooth.
ret = solvers.solve([f1, f2], np.zeros(len(y)), **params)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['niter'], 25)
# Solve with 1 smooth.
ret = solvers.solve([f1], np.zeros(len(y)), **params)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['niter'], 77)
# Solve with 1 non-smooth.
ret = solvers.solve([f2], np.zeros(len(y)), **params)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['niter'], 18)
# Solve with 1 smooth and 2 non-smooth.
ret = solvers.solve([f1, f2, f2], np.zeros(len(y)), **params)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['niter'], 26)
# Solve with 2 smooth and 2 non-smooth.
ret = solvers.solve([f2, f1, f2, f1], np.zeros(len(y)), **params)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['niter'], 25)
# Sanity checks
x0 = np.zeros((4,))
solver.lambda_ = 2.
self.assertRaises(ValueError, solver.pre, [f1, f2], x0)
solver.lambda_ = -2.
self.assertRaises(ValueError, solver.pre, [f1, f2], x0)
f1 = functions.func()
f2 = functions.func()
f3 = functions.func()
solver.lambda_ = 1.
self.assertRaises(ValueError, solver.pre, [f1, f2, f3], x0)
def test_generalized_forward_backward(self):
"""
Test the generalized forward-backward algorithm.
"""
y = [4, 5, 6, 7]
L = 4 # Gradient of the smooth function is Lipschitz continuous.
solver = solvers.generalized_forward_backward(step=.9 / L, lambda_=.8)
params = {'solver': solver, 'verbosity': 'NONE'}
# Functions.
f1 = functions.norm_l1(y=y, lambda_=.7) # Non-smooth.
f2 = functions.norm_l2(y=y, lambda_=L / 2.) # Smooth.
# Solve with 1 smooth and 1 non-smooth.
ret = solvers.solve([f1, f2], np.zeros(len(y)), **params)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['niter'], 25)
# Solve with 1 smooth.
ret = solvers.solve([f1], np.zeros(len(y)), **params)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['niter'], 77)
# Solve with 1 non-smooth.
ret = solvers.solve([f2], np.zeros(len(y)), **params)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['niter'], 18)
# Solve with 1 smooth and 2 non-smooth.
ret = solvers.solve([f1, f2, f2], np.zeros(len(y)), **params)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['niter'], 26)
# Solve with 2 smooth and 2 non-smooth.
ret = solvers.solve([f2, f1, f2, f1], np.zeros(len(y)), **params)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['niter'], 25)
# Sanity checks
x0 = np.zeros((4, ))
solver.lambda_ = 2.
self.assertRaises(ValueError, solver.pre, [f1, f2], x0)
solver.lambda_ = -2.
self.assertRaises(ValueError, solver.pre, [f1, f2], x0)
f1 = functions.func()
f2 = functions.func()
f3 = functions.func()
solver.lambda_ = 1.
self.assertRaises(ValueError, solver.pre, [f1, f2, f3], x0)
def test_primal_dual_solver_comparison(self):
"""
Test that all primal-dual solvers return the same and correct solution.
I had to create this separate function because the primal-dual solvers
were too slow for the problem above.
"""
# Convex functions.
y = np.array([294, 390, 361])
sol = [1., 1., 1.]
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
f1 = functions.norm_l1(y=y)
f2 = functions.norm_l1()
f3 = functions.dummy()
# Solvers.
step = 0.5 / (1 + np.linalg.norm(L, 2))
slvs = []
slvs.append(solvers.mlfbf(step=step))
slvs.append(solvers.projection_based(step=step))
# Compare solutions.
params = {'rtol': 0, 'verbosity': 'NONE', 'maxit': 50}
niters = [50, 50]
for solver, niter in zip(slvs, niters):
x0 = np.zeros(len(y))
if type(solver) is solvers.mlfbf:
ret = solvers.solve([f1, f2, f3], x0, solver, **params)
else:
ret = solvers.solve([f1, f2], x0, solver, **params)
nptest.assert_allclose(ret['sol'], sol)
self.assertEqual(ret['niter'], niter)
self.assertIs(ret['sol'], x0) # The initial value was modified.
def test_primal_dual_solver_comparison(self):
"""
Test that all primal-dual solvers return the same and correct solution.
I had to create this separate function because the primal-dual solvers
were too slow for the problem above.
"""
# Convex functions.
y = np.array([294, 390, 361])
sol = [1., 1., 1.]
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
f1 = functions.norm_l1(y=y)
f2 = functions.norm_l1()
f3 = functions.dummy()
# Solvers.
step = 0.5 / (1 + np.linalg.norm(L, 2))
slvs = []
slvs.append(solvers.mlfbf(step=step))
slvs.append(solvers.projection_based(step=step))
# Compare solutions.
params = {'rtol': 0, 'verbosity': 'NONE', 'maxit': 50}
niters = [50, 50]
for solver, niter in zip(slvs, niters):
x0 = np.zeros(len(y))
if type(solver) is solvers.mlfbf:
ret = solvers.solve([f1, f2, f3], x0, solver, **params)
else:
ret = solvers.solve([f1, f2], x0, solver, **params)
nptest.assert_allclose(ret['sol'], sol)
self.assertEqual(ret['niter'], niter)
self.assertIs(ret['sol'], x0) # The initial value was modified.
def test_projection_based(self):
"""
Test the projection-based solver with arbitrarily selected functions.
"""
x = [0, 0, 0]
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
solver = solvers.projection_based(L=L, step=1.)
params = {'solver': solver, 'verbosity': 'NONE'}
# L1-norm prox and dummy prox.
f = functions.norm_l1(y=np.array([294, 390, 361]))
g = functions.norm_l1()
ret = solvers.solve([f, g], np.array([500, 1000, -400]),
maxit=1000, rtol=None, xtol=0.1, **params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Sanity checks
def x0(): return np.zeros(len(x))
self.assertRaises(ValueError, solver.pre, [f], x0())
solver.lambda_ = 3.
self.assertRaises(ValueError, solver.pre, [f, g], x0())
solver.lambda_ = -3.
self.assertRaises(ValueError, solver.pre, [f, g], x0())
def test_prox_star(self):
n = 10
x = 3 * np.random.randn(n, 1)
f = functions.norm_l1()
f2 = functions.dummy()
f2.prox = lambda x, T: functions._prox_star(f, x, T)
gamma = np.random.rand()
p1 = f.prox(x, gamma)
p2 = functions._prox_star(f2, x, gamma)
np.testing.assert_array_almost_equal(p1, p2)
p1 = f.prox(x, gamma) - x
p2 = -gamma * f2.prox(x / gamma, 1 / gamma)
np.testing.assert_array_almost_equal(p1, p2)
def lasso(A, b, err, nvis, nt3phi, l, maxit):
from pyunlocbox import functions
from pyunlocbox import solvers
x = np.zeros(A.shape[1])
x = x.reshape(-1, 1)
for ty in range(A.shape[1]):
A[nvis:, ty] = np.rad2deg(np.arctan(np.tan(np.deg2rad(A[nvis:, ty]))))
b[nvis:, 0] = np.rad2deg(np.arctan(np.tan(np.deg2rad(b[nvis:, 0]))))
f1 = functions.norm_l1(lambda_=l)
f2 = functions.norm_l2(y=b, A=A)
step = 0.5 / np.linalg.norm(A, ord=2)**2
solver = solvers.forward_backward(step=step)
ret = solvers.solve([f1, f2], x, solver, rtol=1e-6, maxit=maxit)
return ret
def test_norm_l1(self):
"""
Test the norm_l1 derived class.
We test the two methods : eval and prox.
First with default class properties, then custom ones.
"""
f = functions.norm_l1(lambda_=3)
self.assertEqual(f.eval([10, 0]), 30)
self.assertEqual(f.eval(np.array([-10, 0])), 30)
self.assertEqual(f.eval([-3, 4]), 21)
nptest.assert_array_equal(f.prox(np.array([[1, -4], [5, -2]]), 1),
[[0, -1], [2, 0]])
f = functions.norm_l1(tight=False)
self.assertRaises(NotImplementedError, f.prox, [1, 2], 1)
# Let us test the weights
f2 = functions.norm_l1(lambda_=4, w=2)
self.assertEqual(f2.eval([10, 1]), 88)
self.assertEqual(f2.eval([5]), 40)
self.assertEqual(f2.eval([0, -2, 1]), 24)
self.assertEqual(f2.eval(np.array([-10, 0])), 80)
self.assertEqual(f2.eval([-3, 4]), 56)
nptest.assert_array_equal(f2.prox(np.array([[7, -9], [10, -2]]), 1),
[[0, -1], [2, 0]])
# Let us multiple weights
f3 = functions.norm_l1(lambda_=4, w=[1, 2])
self.assertEqual(f3.eval([10, 1]), 48)
nptest.assert_array_equal(f3.prox(np.array([[7, -9], [10, -2]]), 1),
[[3, -1], [6, 0]])
# Let us test y
f3 = functions.norm_l1(lambda_=2, y=[1, 2])
self.assertEqual(f3.eval([10, 1]), 20)
nptest.assert_array_equal(f3.prox(np.array([[2, 5], [-1, -2]]), 1),
[[1, 3], [1, 0]])
# Forward operators (square and non-square).
A = np.eye(3)
f = functions.norm_l1(A=A)
self.assertEqual(f.eval([1, 2, 4]), 7)
nptest.assert_allclose(f.prox([1, 2, 4], 1), [0, 1, 3])
A = np.concatenate([A] * 2, 0)
f = functions.norm_l1(A=A)
self.assertEqual(f.eval([1, 2, 4]), 14)
nptest.assert_allclose(f.prox([1, 2, 4], 1), [-1, 0, 2])
def Experiment(iterable):
M = 0
Omega = np.random.permutation(N)
x_0 = np.zeros(N)
x_0[Omega[0:k]] = np.random.standard_normal(k)
psi = np.ones(N)
Psi = np.diag(psi)
Phi = np.random.randn(n, N)
A = np.dot(Phi, Psi)
y = np.dot(A, x_0)
x = np.zeros(N)
f1 = functions.norm_l1()
f2 = functions.proj_b2(epsilon=0, y=y, A=A, tight=False,\
nu=np.linalg.norm(A, ord=2)**2)
solver = solvers.douglas_rachford(step=1e-2)
ret = solvers.solve([f1, f2], x, solver, rtol=1e-4, maxit=300)
x = ret.get('sol')
residual = (float(LA.norm(x - x_0, 2)) / LA.norm(x_0, 2))**2
if residual <= tolerance:
M += 1
return M
def solve_lasso(A_val, y_val, hparams): #(n,m), (1,m)
if hparams.lasso_solver == 'sklearn':
lasso_est = Lasso(alpha=hparams.lmbd)
lasso_est.fit(A_val.T, y_val.reshape(hparams.num_measurements))
x_hat = lasso_est.coef_
x_hat = np.reshape(x_hat, [-1])
elif hparams.lasso_solver == 'cvxopt':
A_mat = matrix(A_val.T) #[m,n]
y_mat = matrix(y_val.T) ###
x_hat_mat = l1regls(A_mat, y_mat)
x_hat = np.asarray(x_hat_mat)
x_hat = np.reshape(x_hat, [-1]) #[n, ]
elif hparams.lasso_solver == 'pyunlocbox':
tau = hparams.tau
f1 = functions.norm_l1(lambda_=tau)
f2 = functions.norm_l2(y=y_val.T, A=A_val.T)
if hparams.model_type == 'compressing':
if hparams.image_mode == '3D':
A_real = np.random.normal(
size=(int(hparams.num_measurements / 3),
sig_shape))
else:
A_real = np.random.normal(
size=(hparams.num_measurements, sig_shape))
step = 0.5 / np.linalg.norm(A_real, ord=2)**2
else:
step = 0.5 / np.linalg.norm(A_val, ord=2)**2
solver = solvers.forward_backward(step=step)
x0 = np.zeros((sig_shape, 1))
ret = solvers.solve([f1, f2],
x0,
solver,
rtol=1e-4,
maxit=3000)
x_hat_mat = ret['sol']
x_hat = np.asarray(x_hat_mat)
x_hat = np.reshape(x_hat, [-1]) #[n, ]
return x_hat
def test_mlfbf(self):
"""
Test the MLFBF solver with arbitrarily selected functions.
"""
x = [1., 1., 1.]
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
max_step = 1 / (1 + np.linalg.norm(L, 2))
solver = solvers.mlfbf(L=L, step=max_step / 2.)
params = {'solver': solver, 'verbosity': 'NONE'}
def x0():
return np.zeros(len(x))
# L2-norm prox and dummy prox.
f = functions.dummy()
f._prox = lambda x, T: np.maximum(np.zeros(len(x)), x)
g = functions.norm_l2(lambda_=0.5)
h = functions.norm_l2(y=np.array([294, 390, 361]), lambda_=0.5)
ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Same test, but with callable L
solver = solvers.mlfbf(L=lambda x: np.dot(L, x),
Lt=lambda y: np.dot(L.T, y),
d0=np.dot(L, x0()),
step=max_step / 2.)
ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Sanity check
self.assertRaises(ValueError, solver.pre, [f, g], x0())
# Make a second test where the solution is calculated by hand
n = 10
y = np.random.rand(n) * 2
z = np.random.rand(n)
c = 1
delta = (y - z - c)**2 + 4 * (1 + y * z - z * c)
sol = 0.5 * ((y - z - c) + np.sqrt(delta))
class mlog(functions.func):
def __init__(self, z):
super().__init__()
self.z = z
def _eval(self, x):
return -np.sum(np.log(x + self.z))
def _prox(self, x, T):
delta = (x - self.z)**2 + 4 * (T + x * self.z)
sol = 0.5 * (x - self.z + np.sqrt(delta))
return sol
f = functions.norm_l1(lambda_=c)
g = mlog(z=z)
h = functions.norm_l2(lambda_=0.5, y=y)
mu = 1 + 1
step = 1 / mu / 2
solver = solvers.mlfbf(step=step)
ret = solvers.solve([f, g, h],
y.copy(),
solver,
maxit=200,
rtol=0,
verbosity="NONE")
nptest.assert_allclose(ret["sol"], sol, atol=1e-10)
# Make a final test where the function g can not be evaluate
# on the primal variables
y = np.random.rand(3)
y_2 = L.dot(y)
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
x0 = np.zeros(len(y))
f = functions.norm_l1(y=y)
g = functions.norm_l2(lambda_=0.5, y=y_2)
h = functions.norm_l2(y=y, lambda_=0.5)
max_step = 1 / (1 + np.linalg.norm(L, 2))
solver = solvers.mlfbf(L=L, step=max_step / 2.)
ret = solvers.solve([f, g, h], x0, solver, maxit=1000, rtol=0)
np.testing.assert_allclose(ret["sol"], y)
def prox_tv(x,
gamma,
G,
A=None,
At=None,
nu=1,
tol=10e-4,
maxit=200,
use_matrix=True):
r"""
Total Variation proximal operator for graphs.
This function computes the TV proximal operator for graphs. The TV norm
is the one norm of the gradient. The gradient is defined in the
function :meth:`pygsp.graphs.Graph.grad`.
This function requires the PyUNLocBoX to be executed.
This function solves:
:math:`sol = \min_{z} \frac{1}{2} \|x - z\|_2^2 + \gamma \|x\|_{TV}`
Parameters
----------
x: int
Input signal
gamma: ndarray
Regularization parameter
G: graph object
Graphs structure
A: lambda function
Forward operator, this parameter allows to solve the following problem:
:math:`sol = \min_{z} \frac{1}{2} \|x - z\|_2^2 + \gamma \| A x\|_{TV}`
(default = Id)
At: lambda function
Adjoint operator. (default = Id)
nu: float
Bound on the norm of the operator (default = 1)
tol: float
Stops criterion for the loop. The algorithm will stop if :
:math:`\frac{n(t) - n(t - 1)} {n(t)} < tol`
where :math:`n(t) = f(x) + 0.5 \|x-y\|_2^2` is the objective function at iteration :math:`t`
(default = :math:`10e-4`)
maxit: int
Maximum iteration. (default = 200)
use_matrix: bool
If a matrix should be used. (default = True)
Returns
-------
sol: solution
Examples
--------
"""
if A is None:
def A(x):
return x
if At is None:
def At(x):
return x
tight = 0
l1_nu = 2 * G.lmax * nu
if use_matrix:
def l1_a(x):
return G.Diff * A(x)
def l1_at(x):
return G.Diff * At(D.T * x)
else:
def l1_a(x):
return G.grad(A(x))
def l1_at(x):
return G.div(x)
functions, _ = _import_pyunlocbox()
functions.norm_l1(x,
gamma,
A=l1_a,
At=l1_at,
tight=tight,
maxit=maxit,
verbose=verbose,
tol=tol)
def test_forward_backward_fista(self):
"""
Test forward-backward splitting solver with fista acceleration,
solving problems with L1-norm, L2-norm, and dummy functions.
"""
y = [4., 5., 6., 7.]
solver = solvers.forward_backward(accel=acceleration.fista())
param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 60)
# Dummy prox and L2-norm gradient.
f1 = functions.dummy()
f2 = functions.norm_l2(y=y, lambda_=0.6)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 84)
# L2-norm prox and L2-norm gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.norm_l2(y=y)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y, rtol=1e-2)
self.assertEqual(ret['crit'], 'MAXIT')
self.assertEqual(ret['niter'], 200)
# L1-norm prox and dummy gradient.
f1 = functions.norm_l1(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 6)
# Dummy prox and L1-norm gradient. As L1-norm possesses no gradient,
# the algorithm exchanges the functions : exact same solution.
f1 = functions.dummy()
f2 = functions.norm_l1(y=y)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 6)
# L1-norm prox and L1-norm gradient. L1-norm possesses no gradient.
f1 = functions.norm_l1(y=y)
f2 = functions.norm_l1(y=y)
self.assertRaises(ValueError, solvers.solve,
[f1, f2], np.zeros(len(y)), **param)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 10)
def test_forward_backward_fista(self):
"""
Test forward-backward splitting solver with fista acceleration,
solving problems with L1-norm, L2-norm, and dummy functions.
"""
y = [4., 5., 6., 7.]
solver = solvers.forward_backward(accel=acceleration.fista())
param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'}
# L2-norm prox and dummy gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 60)
# Dummy prox and L2-norm gradient.
f1 = functions.dummy()
f2 = functions.norm_l2(y=y, lambda_=0.6)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 84)
# L2-norm prox and L2-norm gradient.
f1 = functions.norm_l2(y=y)
f2 = functions.norm_l2(y=y)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y, rtol=1e-2)
self.assertEqual(ret['crit'], 'MAXIT')
self.assertEqual(ret['niter'], 200)
# L1-norm prox and dummy gradient.
f1 = functions.norm_l1(y=y)
f2 = functions.dummy()
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 6)
# Dummy prox and L1-norm gradient. As L1-norm possesses no gradient,
# the algorithm exchanges the functions : exact same solution.
f1 = functions.dummy()
f2 = functions.norm_l1(y=y)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 6)
# L1-norm prox and L1-norm gradient. L1-norm possesses no gradient.
f1 = functions.norm_l1(y=y)
f2 = functions.norm_l1(y=y)
self.assertRaises(ValueError, solvers.solve, [f1, f2],
np.zeros(len(y)), **param)
# L1-norm prox and L2-norm gradient.
f1 = functions.norm_l1(y=y, lambda_=1.0)
f2 = functions.norm_l2(y=y, lambda_=0.8)
ret = solvers.solve([f1, f2], np.zeros(len(y)), **param)
nptest.assert_allclose(ret['sol'], y)
self.assertEqual(ret['crit'], 'RTOL')
self.assertEqual(ret['niter'], 10)
def prox_tv(x, gamma, G, A=None, At=None, nu=1, tol=10e-4, maxit=200, use_matrix=True):
r"""
Total Variation proximal operator for graphs.
This function computes the TV proximal operator for graphs. The TV norm
is the one norm of the gradient. The gradient is defined in the
function :meth:`pygsp.graphs.Graph.grad`.
This function requires the PyUNLocBoX to be executed.
This function solves:
:math:`sol = \min_{z} \frac{1}{2} \|x - z\|_2^2 + \gamma \|x\|_{TV}`
Parameters
----------
x: int
Input signal
gamma: ndarray
Regularization parameter
G: graph object
Graphs structure
A: lambda function
Forward operator, this parameter allows to solve the following problem:
:math:`sol = \min_{z} \frac{1}{2} \|x - z\|_2^2 + \gamma \| A x\|_{TV}`
(default = Id)
At: lambda function
Adjoint operator. (default = Id)
nu: float
Bound on the norm of the operator (default = 1)
tol: float
Stops criterion for the loop. The algorithm will stop if :
:math:`\frac{n(t) - n(t - 1)} {n(t)} < tol`
where :math:`n(t) = f(x) + 0.5 \|x-y\|_2^2` is the objective function at iteration :math:`t`
(default = :math:`10e-4`)
maxit: int
Maximum iteration. (default = 200)
use_matrix: bool
If a matrix should be used. (default = True)
Returns
-------
sol: solution
Examples
--------
"""
if A is None:
def A(x):
return x
if At is None:
def At(x):
return x
tight = 0
l1_nu = 2 * G.lmax * nu
if use_matrix:
def l1_a(x):
return G.Diff * A(x)
def l1_at(x):
return G.Diff * At(D.T * x)
else:
def l1_a(x):
return G.grad(A(x))
def l1_at(x):
return G.div(x)
functions, _ = _import_pyunlocbox()
functions.norm_l1(x, gamma, A=l1_a, At=l1_at, tight=tight, maxit=maxit, verbose=verbose, tol=tol)
