def test_backtracking(self): """ Test forward-backward splitting solver with backtracking, solving problems with L1-norm, L2-norm, and dummy functions. """ # Test constructor sanity a = acceleration.backtracking() self.assertRaises(ValueError, a.__init__, 2.) self.assertRaises(ValueError, a.__init__, -2.) y = [4., 5., 6., 7.] accel = acceleration.backtracking() step = 10 # Make sure backtracking is called solver = solvers.forward_backward(accel=accel, step=step) param = {'solver': solver, 'atol': 1e-32, 'verbosity': 'NONE'} # L2-norm prox and dummy gradient. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'ATOL') self.assertEqual(ret['niter'], 13) # L1-norm prox and L2-norm gradient. f1 = functions.norm_l1(y=y, lambda_=1.0) f2 = functions.norm_l2(y=y, lambda_=0.8) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'ATOL') self.assertLessEqual(ret['niter'], 4) # win64 takes one iteration
def test_backtracking(self): """ Test forward-backward splitting solver with backtracking, solving problems with L1-norm, L2-norm, and dummy functions. """ # Test constructor sanity a = acceleration.backtracking() self.assertRaises(ValueError, a.__init__, 2.) self.assertRaises(ValueError, a.__init__, -2.) y = [4., 5., 6., 7.] accel = acceleration.backtracking() step = 10 # Make sure backtracking is called solver = solvers.forward_backward(accel=accel, step=step) param = {'solver': solver, 'atol': 1e-32, 'verbosity': 'NONE'} # L2-norm prox and dummy gradient. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'ATOL') self.assertEqual(ret['niter'], 13) # L1-norm prox and L2-norm gradient. f1 = functions.norm_l1(y=y, lambda_=1.0) f2 = functions.norm_l2(y=y, lambda_=0.8) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'ATOL') self.assertEqual(ret['niter'], 4)
def test_forward_backward_fista_backtracking(self): """ Test forward-backward splitting solver with fista acceleration and backtracking, solving problems with L1-norm, L2-norm, and dummy functions. """ y = [4., 5., 6., 7.] accel = acceleration.fista_backtracking() solver = solvers.forward_backward(accel=accel) param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'} # L2-norm prox and dummy gradient. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 60) # L1-norm prox and L2-norm gradient. f1 = functions.norm_l1(y=y, lambda_=1.0) f2 = functions.norm_l2(y=y, lambda_=0.8) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 3)
def test_mlfbf(self): """ Test the MLFBF solver with arbitrarily selected functions. """ x = [1., 1., 1.] L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]]) max_step = 1 / (1 + np.linalg.norm(L, 2)) solver = solvers.mlfbf(L=L, step=max_step / 2.) params = {'solver': solver, 'verbosity': 'NONE'} def x0(): return np.zeros(len(x)) # L2-norm prox and dummy prox. f = functions.dummy() f._prox = lambda x, T: np.maximum(np.zeros(len(x)), x) g = functions.norm_l2(lambda_=0.5) h = functions.norm_l2(y=np.array([294, 390, 361]), lambda_=0.5) ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params) nptest.assert_allclose(ret['sol'], x, rtol=1e-5) # Same test, but with callable L solver = solvers.mlfbf(L=lambda x: np.dot(L, x), Lt=lambda y: np.dot(L.T, y), d0=np.dot(L, x0()), step=max_step / 2.) ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params) nptest.assert_allclose(ret['sol'], x, rtol=1e-5) # Sanity check self.assertRaises(ValueError, solver.pre, [f, g], x0())
def test_douglas_rachford(self): """ Test douglas-rachford solver with L1-norm, L2-norm and dummy functions. """ y = [4, 5, 6, 7] solver = solvers.douglas_rachford() param = {'solver': solver, 'verbosity': 'NONE'} # L2-norm prox and dummy prox. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 35) # L2-norm prox and L1-norm prox. f1 = functions.norm_l2(y=y) f2 = functions.norm_l1(y=y) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 4) # Sanity checks x0 = np.zeros((4,)) solver.lambda_ = 2. self.assertRaises(ValueError, solver.pre, [f1, f2], x0) solver.lambda_ = -2. self.assertRaises(ValueError, solver.pre, [f1, f2], x0) self.assertRaises(ValueError, solver.pre, [f1, f2, f1], x0)
def test_forward_backward(self): """ Test forward-backward splitting algorithm without acceleration, and with L1-norm, L2-norm, and dummy functions. """ y = [4., 5., 6., 7.] solver = solvers.forward_backward(accel=acceleration.dummy()) param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'} # L2-norm prox and dummy gradient. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 35) # L1-norm prox and L2-norm gradient. f1 = functions.norm_l1(y=y, lambda_=1.0) f2 = functions.norm_l2(y=y, lambda_=0.8) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 4) # Sanity check f3 = functions.dummy() x0 = np.zeros((4,)) self.assertRaises(ValueError, solver.pre, [f1, f2, f3], x0)
def test_douglas_rachford(self): """ Test douglas-rachford solver with L1-norm, L2-norm and dummy functions. """ y = [4, 5, 6, 7] solver = solvers.douglas_rachford() param = {'solver': solver, 'verbosity': 'NONE'} # L2-norm prox and dummy prox. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 35) # L2-norm prox and L1-norm prox. f1 = functions.norm_l2(y=y) f2 = functions.norm_l1(y=y) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 4) # Sanity checks x0 = np.zeros((4, )) solver.lambda_ = 2. self.assertRaises(ValueError, solver.pre, [f1, f2], x0) solver.lambda_ = -2. self.assertRaises(ValueError, solver.pre, [f1, f2], x0) self.assertRaises(ValueError, solver.pre, [f1, f2, f1], x0)
def test_forward_backward(self): """ Test forward-backward splitting algorithm without acceleration, and with L1-norm, L2-norm, and dummy functions. """ y = [4., 5., 6., 7.] solver = solvers.forward_backward(accel=acceleration.dummy()) param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'} # L2-norm prox and dummy gradient. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 35) # L1-norm prox and L2-norm gradient. f1 = functions.norm_l1(y=y, lambda_=1.0) f2 = functions.norm_l2(y=y, lambda_=0.8) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 4) # Sanity check f3 = functions.dummy() x0 = np.zeros((4, )) self.assertRaises(ValueError, solver.pre, [f1, f2, f3], x0)
def test_primal_dual_solver_comparison(self): """ Test that all primal-dual solvers return the same and correct solution. I had to create this separate function because the primal-dual solvers were too slow for the problem above. """ # Convex functions. y = np.random.randn(3) L = np.random.randn(4, 3) sol = y y2 = L.dot(y) f1 = functions.norm_l1(y=y) f2 = functions.norm_l2(y=y2) f3 = functions.dummy() # Solvers. step = 0.5 / (1 + np.linalg.norm(L, 2)) slvs = [] slvs.append(solvers.mlfbf(step=step, L=L)) slvs.append(solvers.projection_based(step=step, L=L)) # Compare solutions. niter = 1000 params = {'rtol': 0, 'verbosity': 'NONE', 'maxit': niter} for solver in slvs: x0 = np.zeros(len(y)) if type(solver) is solvers.mlfbf: ret = solvers.solve([f1, f2, f3], x0, solver, **params) else: ret = solvers.solve([f1, f2], x0, solver, **params) nptest.assert_allclose(ret['sol'], sol) self.assertEqual(ret['niter'], niter) # The initial value was not modified. nptest.assert_array_equal(x0, np.zeros(len(y))) if type(solver) is solvers.mlfbf: ret = solvers.solve([f1, f2, f3], x0, solver, inplace=True, **params) else: ret = solvers.solve([f1, f2], x0, solver, inplace=True, **params) # The initial value was modified. self.assertIs(ret['sol'], x0) nptest.assert_allclose(ret['sol'], sol)
def test_generalized_forward_backward(self): """ Test the generalized forward-backward algorithm. """ y = [4, 5, 6, 7] L = 4 # Gradient of the smooth function is Lipschitz continuous. solver = solvers.generalized_forward_backward(step=.9 / L, lambda_=.8) params = {'solver': solver, 'verbosity': 'NONE'} # Functions. f1 = functions.norm_l1(y=y, lambda_=.7) # Non-smooth. f2 = functions.norm_l2(y=y, lambda_=L / 2.) # Smooth. # Solve with 1 smooth and 1 non-smooth. ret = solvers.solve([f1, f2], np.zeros(len(y)), **params) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['niter'], 25) # Solve with 1 smooth. ret = solvers.solve([f1], np.zeros(len(y)), **params) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['niter'], 77) # Solve with 1 non-smooth. ret = solvers.solve([f2], np.zeros(len(y)), **params) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['niter'], 18) # Solve with 1 smooth and 2 non-smooth. ret = solvers.solve([f1, f2, f2], np.zeros(len(y)), **params) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['niter'], 26) # Solve with 2 smooth and 2 non-smooth. ret = solvers.solve([f2, f1, f2, f1], np.zeros(len(y)), **params) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['niter'], 25) # Sanity checks x0 = np.zeros((4,)) solver.lambda_ = 2. self.assertRaises(ValueError, solver.pre, [f1, f2], x0) solver.lambda_ = -2. self.assertRaises(ValueError, solver.pre, [f1, f2], x0) f1 = functions.func() f2 = functions.func() f3 = functions.func() solver.lambda_ = 1. self.assertRaises(ValueError, solver.pre, [f1, f2, f3], x0)
def test_generalized_forward_backward(self): """ Test the generalized forward-backward algorithm. """ y = [4, 5, 6, 7] L = 4 # Gradient of the smooth function is Lipschitz continuous. solver = solvers.generalized_forward_backward(step=.9 / L, lambda_=.8) params = {'solver': solver, 'verbosity': 'NONE'} # Functions. f1 = functions.norm_l1(y=y, lambda_=.7) # Non-smooth. f2 = functions.norm_l2(y=y, lambda_=L / 2.) # Smooth. # Solve with 1 smooth and 1 non-smooth. ret = solvers.solve([f1, f2], np.zeros(len(y)), **params) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['niter'], 25) # Solve with 1 smooth. ret = solvers.solve([f1], np.zeros(len(y)), **params) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['niter'], 77) # Solve with 1 non-smooth. ret = solvers.solve([f2], np.zeros(len(y)), **params) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['niter'], 18) # Solve with 1 smooth and 2 non-smooth. ret = solvers.solve([f1, f2, f2], np.zeros(len(y)), **params) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['niter'], 26) # Solve with 2 smooth and 2 non-smooth. ret = solvers.solve([f2, f1, f2, f1], np.zeros(len(y)), **params) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['niter'], 25) # Sanity checks x0 = np.zeros((4, )) solver.lambda_ = 2. self.assertRaises(ValueError, solver.pre, [f1, f2], x0) solver.lambda_ = -2. self.assertRaises(ValueError, solver.pre, [f1, f2], x0) f1 = functions.func() f2 = functions.func() f3 = functions.func() solver.lambda_ = 1. self.assertRaises(ValueError, solver.pre, [f1, f2, f3], x0)
def test_solver_comparison(self): """ Test that all solvers return the same and correct solution. """ # Convex functions. y = [1, 0, 0.1, 8, -6.5, 0.2, 0.004, 0.01] sol = [0.75, 0, 0, 7.75, -6.25, 0, 0, 0] w1, w2 = .8, .4 f1 = functions.norm_l2(y=y, lambda_=w1 / 2.) # Smooth. f2 = functions.norm_l1(lambda_=w2 / 2.) # Non-smooth. # Solvers. L = w1 # Lipschitz continuous gradient. step = 1. / L lambda_ = 0.5 params = {'step': step, 'lambda_': lambda_} slvs = [] slvs.append(solvers.forward_backward(accel=acceleration.dummy(), step=step)) slvs.append(solvers.douglas_rachford(**params)) slvs.append(solvers.generalized_forward_backward(**params)) # Compare solutions. params = {'rtol': 1e-14, 'verbosity': 'NONE', 'maxit': 1e4} niters = [2, 61, 26] for solver, niter in zip(slvs, niters): x0 = np.zeros(len(y)) ret = solvers.solve([f1, f2], x0, solver, **params) nptest.assert_allclose(ret['sol'], sol) self.assertEqual(ret['niter'], niter) self.assertIs(ret['sol'], x0) # The initial value was modified.
def test_regularized_nonlinear(self): """ Test gradient descent solver with regularized non-linear acceleration, solving problems with L2-norm functions. """ dim = 25 np.random.seed(0) x0 = np.random.rand(dim) xstar = np.random.rand(dim) x0 = xstar + 5. * (x0 - xstar) / np.linalg.norm(x0 - xstar) A = np.random.rand(dim, dim) step = 1 / np.linalg.norm(np.dot(A.T, A)) accel = acceleration.regularized_nonlinear(k=5) solver = solvers.gradient_descent(step=step, accel=accel) param = {'solver': solver, 'rtol': 0, 'maxit': 200, 'verbosity': 'NONE'} # L2-norm prox and dummy gradient. f1 = functions.norm_l2(lambda_=0.5, A=A, y=np.dot(A, xstar)) f2 = functions.dummy() ret = solvers.solve([f1, f2], x0, **param) pctdiff = 100 * np.sum((xstar - ret['sol'])**2) / np.sum(xstar**2) nptest.assert_array_less(pctdiff, 1.91) # Sanity checks accel = acceleration.regularized_nonlinear() self.assertRaises(ValueError, accel.__init__, 10, ['not', 'good']) self.assertRaises(ValueError, accel.__init__, 10, 'nope')
def test_acceleration_comparison(self): """ Test that all solvers return the same and correct solution. """ # Convex functions. y = [1, 0, 0.1, 8, -6.5, 0.2, 0.004, 0.01] sol = [0.75, 0, 0, 7.75, -6.25, 0, 0, 0] w1, w2 = .8, .4 f1 = functions.norm_l2(y=y, lambda_=w1 / 2.) # Smooth. f2 = functions.norm_l1(lambda_=w2 / 2.) # Non-smooth. # Solvers. L = w1 # Lipschitz continuous gradient. step = 1. / L slvs = [] slvs.append( solvers.forward_backward(accel=acceleration.dummy(), step=step)) slvs.append( solvers.forward_backward(accel=acceleration.fista(), step=step)) slvs.append( solvers.forward_backward( accel=acceleration.fista_backtracking(eta=.999), step=step)) # Compare solutions. params = {'rtol': 1e-14, 'verbosity': 'NONE', 'maxit': 1e4} niters = [2, 2, 6] for solver, niter in zip(slvs, niters): x0 = np.zeros(len(y)) ret = solvers.solve([f1, f2], x0, solver, **params) nptest.assert_allclose(ret['sol'], sol) self.assertEqual(ret['niter'], niter)
def test_projection_based(self): """ Test the projection-based solver with arbitrarily selected functions. """ x = [0, 0, 0] L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]]) solver = solvers.projection_based(L=L, step=1.) params = {'solver': solver, 'verbosity': 'NONE'} # L1-norm prox and dummy prox. f = functions.norm_l1(y=np.array([294, 390, 361])) g = functions.norm_l1() ret = solvers.solve([f, g], np.array([500, 1000, -400]), maxit=1000, rtol=None, xtol=0.1, **params) nptest.assert_allclose(ret['sol'], x, rtol=1e-5) # Sanity checks def x0(): return np.zeros(len(x)) self.assertRaises(ValueError, solver.pre, [f], x0()) solver.lambda_ = 3. self.assertRaises(ValueError, solver.pre, [f, g], x0()) solver.lambda_ = -3. self.assertRaises(ValueError, solver.pre, [f, g], x0())
def test_solver_comparison(self): """ Test that all solvers return the same and correct solution. """ # Convex functions. y = [1, 0, 0.1, 8, -6.5, 0.2, 0.004, 0.01] sol = [0.75, 0, 0, 7.75, -6.25, 0, 0, 0] w1, w2 = .8, .4 f1 = functions.norm_l2(y=y, lambda_=w1 / 2.) # Smooth. f2 = functions.norm_l1(lambda_=w2 / 2.) # Non-smooth. # Solvers. L = w1 # Lipschitz continuous gradient. step = 1. / L lambda_ = 0.5 params = {'step': step, 'lambda_': lambda_} slvs = [] slvs.append( solvers.forward_backward(accel=acceleration.dummy(), step=step)) slvs.append(solvers.douglas_rachford(**params)) slvs.append(solvers.generalized_forward_backward(**params)) # Compare solutions. params = {'rtol': 1e-14, 'verbosity': 'NONE', 'maxit': 1e4} niters = [2, 61, 26] for solver, niter in zip(slvs, niters): x0 = np.zeros(len(y)) ret = solvers.solve([f1, f2], x0, solver, **params) nptest.assert_allclose(ret['sol'], sol) self.assertEqual(ret['niter'], niter) self.assertIs(ret['sol'], x0) # The initial value was modified.
def test_acceleration_comparison(self): """ Test that all solvers return the same and correct solution. """ # Convex functions. y = [1, 0, 0.1, 8, -6.5, 0.2, 0.004, 0.01] sol = [0.75, 0, 0, 7.75, -6.25, 0, 0, 0] w1, w2 = .8, .4 f1 = functions.norm_l2(y=y, lambda_=w1 / 2.) # Smooth. f2 = functions.norm_l1(lambda_=w2 / 2.) # Non-smooth. # Solvers. L = w1 # Lipschitz continuous gradient. step = 1. / L slvs = [] slvs.append(solvers.forward_backward(accel=acceleration.dummy(), step=step)) slvs.append(solvers.forward_backward(accel=acceleration.fista(), step=step)) slvs.append(solvers.forward_backward( accel=acceleration.fista_backtracking(eta=.999), step=step)) # Compare solutions. params = {'rtol': 1e-14, 'verbosity': 'NONE', 'maxit': 1e4} niters = [2, 2, 6] for solver, niter in zip(slvs, niters): x0 = np.zeros(len(y)) ret = solvers.solve([f1, f2], x0, solver, **params) nptest.assert_allclose(ret['sol'], sol) self.assertEqual(ret['niter'], niter)
def test_regularized_nonlinear(self): """ Test gradient descent solver with regularized non-linear acceleration, solving problems with L2-norm functions. """ dim = 25 np.random.seed(0) x0 = np.random.rand(dim) xstar = np.random.rand(dim) x0 = xstar + 5. * (x0 - xstar) / np.linalg.norm(x0 - xstar) A = np.random.rand(dim, dim) step = 1 / np.linalg.norm(np.dot(A.T, A)) accel = acceleration.regularized_nonlinear(k=5) solver = solvers.gradient_descent(step=step, accel=accel) param = { 'solver': solver, 'rtol': 0, 'maxit': 200, 'verbosity': 'NONE' } # L2-norm prox and dummy gradient. f1 = functions.norm_l2(lambda_=0.5, A=A, y=np.dot(A, xstar)) f2 = functions.dummy() ret = solvers.solve([f1, f2], x0, **param) pctdiff = 100 * np.sum((xstar - ret['sol'])**2) / np.sum(xstar**2) nptest.assert_array_less(pctdiff, 1.91) # Sanity checks accel = acceleration.regularized_nonlinear() self.assertRaises(ValueError, accel.__init__, 10, ['not', 'good']) self.assertRaises(ValueError, accel.__init__, 10, 'nope')
def optGFB(D, GK0, triangles, fxy, E, maxit, x1, step, tau1, tau2, tau3, flag): #### N = len(E) f2 = functions.func() f2._eval = lambda x: 0 f2._prox = lambda x, T: np.clip(x, 0, 0.7) # ### T1 = Nodeneighbor(triangles, N) g = lambda x: Ggradient(x, T1) f3 = functions.norm_l1(A=g, At=None, dim=1, y=np.zeros(N), lambda_=tau2) #['EVAL', 'PROX'] ###### yy2 = np.ravel(fxy) #### N_cov = np.zeros((2 * N, 2 * N)) for j in np.arange(0, 2 * N, 2): N_cov[j, j] = 3 N_cov[j + 1, j + 1] = 1 gamma = GK0 @ N_cov @ GK0.T # gamma += np.eye(gamma.shape[1]) * 1e-5 gamma_inv = np.linalg.inv(gamma) f8 = functions.func(lambda_=tau1) f8._eval = lambda x: (yy2 - D @ x).T @ gamma_inv @ (yy2 - D @ x) * 1e+7 f8._grad = lambda x: -D.T @ gamma_inv @ (yy2 - D @ x) * 1e+7 ####### #step = 0.08#0.5 /tau1# (np.linalg.norm(func(x0),2)**2/np.linalg.norm(x0,2)**2) #0.5/tau#2e3/scale solver2 = solvers.generalized_forward_backward( step=step * 0.2 ) #generalized_forward_backward(step=step*0.1)#step*0.1)douglas_rachford # without f3 -->singular matrix ret2 = solvers.solve([f8, f3, f2], x1, solver2, rtol=1e-15, maxit=maxit) objective = np.array(ret2['objective']) sol = ret2['sol'] import matplotlib.pyplot as plt if flag == 1: _ = plt.figure(figsize=(10, 4)) _ = plt.subplot(121) _ = plt.plot(E, 'o', label='Original E') #-np.ones(N)*0.1 _ = plt.plot(ret2['sol'], 'xr', label='Reconstructed E') _ = plt.grid(True) _ = plt.title('Achieved reconstruction') _ = plt.legend(numpoints=1) _ = plt.xlabel('Signal dimension number') _ = plt.ylabel('Signal value') _ = plt.subplot(122) _ = plt.semilogy(objective[:, 0], '-.', label='l2-norm') # _ = plt.semilogy(np.sum(objective, axis=1), label='Global objective') _ = plt.grid(True) _ = plt.title('Convergence') _ = plt.legend(numpoints=1) _ = plt.xlabel('Iteration number') _ = plt.ylabel('Objective function value') _ = plt.show() return sol
def test_primal_dual_solver_comparison(self): """ Test that all primal-dual solvers return the same and correct solution. I had to create this separate function because the primal-dual solvers were too slow for the problem above. """ # Convex functions. y = np.array([294, 390, 361]) sol = [1., 1., 1.] L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]]) f1 = functions.norm_l1(y=y) f2 = functions.norm_l1() f3 = functions.dummy() # Solvers. step = 0.5 / (1 + np.linalg.norm(L, 2)) slvs = [] slvs.append(solvers.mlfbf(step=step)) slvs.append(solvers.projection_based(step=step)) # Compare solutions. params = {'rtol': 0, 'verbosity': 'NONE', 'maxit': 50} niters = [50, 50] for solver, niter in zip(slvs, niters): x0 = np.zeros(len(y)) if type(solver) is solvers.mlfbf: ret = solvers.solve([f1, f2, f3], x0, solver, **params) else: ret = solvers.solve([f1, f2], x0, solver, **params) nptest.assert_allclose(ret['sol'], sol) self.assertEqual(ret['niter'], niter) self.assertIs(ret['sol'], x0) # The initial value was modified.
def lasso(A, b, err, nvis, nt3phi, l, maxit): from pyunlocbox import functions from pyunlocbox import solvers x = np.zeros(A.shape[1]) x = x.reshape(-1, 1) for ty in range(A.shape[1]): A[nvis:, ty] = np.rad2deg(np.arctan(np.tan(np.deg2rad(A[nvis:, ty])))) b[nvis:, 0] = np.rad2deg(np.arctan(np.tan(np.deg2rad(b[nvis:, 0])))) f1 = functions.norm_l1(lambda_=l) f2 = functions.norm_l2(y=b, A=A) step = 0.5 / np.linalg.norm(A, ord=2)**2 solver = solvers.forward_backward(step=step) ret = solvers.solve([f1, f2], x, solver, rtol=1e-6, maxit=maxit) return ret
def inner_solve(self, x, y0, Lyy=1, r2=functions.dummy(), rtol=1e-9, maxit=100000, verbosity='NONE'): """ Solve min_y f(x,y) + r_2(y) Input ----- x - array y0 - initial iterate Lyy - Lipschitz constant of f r2 - pyunlocbox function object rtol - stopping criterion for solver maxit - max iteration count for inner solve verbosity - level of verbosity for inner solver Output ------ y - array """ # get pyunlocbox function object for f(x,.) f = self.misfit(x=x) # setup solver accel = acceleration.fista_backtracking() solver = solvers.forward_backward(step=1 / Lyy, accel=accel) # run solver results = solvers.solve([f, r2], y0, solver, rtol=rtol, maxit=maxit, verbosity=verbosity) # return result y = results['sol'] return y
def test_gradient_descent(self): """ Test gradient descent solver with l2-norms in the objective. """ y = [4., 5., 6., 7.] A = np.array([[1., 1., 1., 0.], [0., 1., 1., 1.], [0., 1., 0., 0.], [1., 0., 0., 1.]]) sol = np.array([0.28846154, 0.11538462, 1.23076923, 1.78846154]) step = 0.5 / (np.linalg.norm(A) + 1.) solver = solvers.gradient_descent(step=step) param = {'solver': solver, 'rtol': 0, 'verbosity': 'NONE'} f1 = functions.norm_l2(y=y) f2 = functions.norm_l2(A=A) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], sol) self.assertEqual(ret['crit'], 'MAXIT') self.assertEqual(ret['niter'], 200)
def Experiment(iterable): M = 0 Omega = np.random.permutation(N) x_0 = np.zeros(N) x_0[Omega[0:k]] = np.random.standard_normal(k) psi = np.ones(N) Psi = np.diag(psi) Phi = np.random.randn(n, N) A = np.dot(Phi, Psi) y = np.dot(A, x_0) x = np.zeros(N) f1 = functions.norm_l1() f2 = functions.proj_b2(epsilon=0, y=y, A=A, tight=False,\ nu=np.linalg.norm(A, ord=2)**2) solver = solvers.douglas_rachford(step=1e-2) ret = solvers.solve([f1, f2], x, solver, rtol=1e-4, maxit=300) x = ret.get('sol') residual = (float(LA.norm(x - x_0, 2)) / LA.norm(x_0, 2))**2 if residual <= tolerance: M += 1 return M
def solve_lasso(A_val, y_val, hparams): #(n,m), (1,m) if hparams.lasso_solver == 'sklearn': lasso_est = Lasso(alpha=hparams.lmbd) lasso_est.fit(A_val.T, y_val.reshape(hparams.num_measurements)) x_hat = lasso_est.coef_ x_hat = np.reshape(x_hat, [-1]) elif hparams.lasso_solver == 'cvxopt': A_mat = matrix(A_val.T) #[m,n] y_mat = matrix(y_val.T) ### x_hat_mat = l1regls(A_mat, y_mat) x_hat = np.asarray(x_hat_mat) x_hat = np.reshape(x_hat, [-1]) #[n, ] elif hparams.lasso_solver == 'pyunlocbox': tau = hparams.tau f1 = functions.norm_l1(lambda_=tau) f2 = functions.norm_l2(y=y_val.T, A=A_val.T) if hparams.model_type == 'compressing': if hparams.image_mode == '3D': A_real = np.random.normal( size=(int(hparams.num_measurements / 3), sig_shape)) else: A_real = np.random.normal( size=(hparams.num_measurements, sig_shape)) step = 0.5 / np.linalg.norm(A_real, ord=2)**2 else: step = 0.5 / np.linalg.norm(A_val, ord=2)**2 solver = solvers.forward_backward(step=step) x0 = np.zeros((sig_shape, 1)) ret = solvers.solve([f1, f2], x0, solver, rtol=1e-4, maxit=3000) x_hat_mat = ret['sol'] x_hat = np.asarray(x_hat_mat) x_hat = np.reshape(x_hat, [-1]) #[n, ] return x_hat
def main_tv(hparams): ## === Set up=== ## # Printer setup #sys.stdout = open(hparams.text_file_path, 'w') # Get inputs if hparams.image_mode == '1D': x_real = np.array(load_1D(hparams.path, hparams.img_name)).astype( np.float32) #[4096,1] elif hparams.image_mode == '2D': x_real = np.array(load_2D(hparams.path, hparams.img_name)).astype( np.float32) #[64,64] elif hparams.image_mode == '3D': x_real = np.array( load_img(hparams.path, hparams.img_name, hparams.decoder_type)).astype( np.float32) #[178,218,3] / [224,224,3] # Initialization #np.random.seed(7) sig_shape = x_real.shape[0] * x_real.shape[ 1] #n = 4096*1 or 64*64 or 178*218 or 224*224 random_vector = None #initialization A = None #initialization selection_mask = None #initialization random_arr = random_flip(sig_shape) #initialization #[n,] mask = None #initialization # Get measurement matirx if hparams.model_type == 'denoising' or hparams.model_type == 'compressing': if hparams.type_measurements == 'random': #compressed sensing if hparams.image_mode != '3D': A = np.random.randn(hparams.num_measurements, sig_shape).astype(np.float32) #[m,n] noise_shape = [hparams.num_measurements, 1] #[m,1] else: A = np.random.randn(int(hparams.num_measurements / 3), sig_shape).astype(np.float32) #[m,n] noise_shape = [int(hparams.num_measurements / 3), 1] #[m,1] elif hparams.type_measurements == 'identity': #denoising A = np.identity(sig_shape).astype(np.float32) #[n,n] noise_shape = [sig_shape, 1] #[n,1] observ_noise = hparams.noise_level * np.random.randn( noise_shape[0], noise_shape[1]) #[n,1] elif hparams.type_measurements == 'circulant': #compressed sensing if hparams.image_mode != '3D': random_vector = np.random.normal(size=sig_shape) #[n,] selection_mask = create_A_selection( sig_shape, hparams.num_measurements) #[1,n] else: random_vector = np.random.normal(size=sig_shape) #[n,] selection_mask = create_A_selection( sig_shape, int(hparams.num_measurements / 3)) #[1,n] def circulant_np(signal_vector, random_arr_p=random_arr.reshape(-1, 1), random_vector_p=random_vector.reshape(-1, 1), selection_mask_p=selection_mask.reshape(-1, 1)): #step 0: Flip signal_vector = signal_vector * random_arr_p #[n,1] * [n,1] -> [n,1] #step 1: F^{-1} @ x r1 = ifft(signal_vector) #[n,1] #step 2: Diag() @ F^{-1} @ x Ft = fft(random_vector_p) #[n,1] r2 = np.multiply(r1, Ft) #[n,1] * [n,1] -> [n,1] #step 3: F @ Diag() @ F^{-1} @ x compressive = fft(r2) #[n,1] #step 4: R_{omega} @ C_{t} @ D){epsilon} compressive = compressive.real #[n,1] select_compressive = compressive * selection_mask_p #[n,1] * [n,1] -> [n,1] return select_compressive elif hparams.model_type == 'inpainting': if hparams.image_mode == '1D': mask = load_mask('Masks', hparams.mask_name_1D, hparams.image_mode, hparams.decoder_type) #[n,1] elif hparams.image_mode == '2D' or hparams.image_mode == '3D': mask = load_mask('Masks', hparams.mask_name_2D, hparams.image_mode, hparams.decoder_type) #[n,n] ## === TV norm === ## if hparams.decoder_type == 'tv_norm': # Construct observation and perform reconstruction if hparams.model_type == 'inpainting': # measurements and observation g = lambda x: mask * x #[4096,1] * [4096,1] / [178,218,3] * [178,218,3] y_real = g(x_real) #[4096,1] / [178,218,3] # tv norm if hparams.image_mode == '1D': f1 = functions.norm_tv(dim=1) elif hparams.image_mode == '2D': f1 = functions.norm_tv(dim=2) elif hparams.image_mode == '3D': f1 = functions.norm_tv(dim=3) # L2 norm tau = hparams.tau f2 = functions.norm_l2(y=y_real, A=g, lambda_=tau) # optimisation solver = solvers.forward_backward(step=0.5 / tau) x0 = np.array(y_real) # Make a copy to preserve im_masked. ret = solvers.solve([f1, f2], x0, solver, maxit=3000) #output = ret['sol'] # output out_img = ret['sol'] #[4096,1] / [178,218,3] elif hparams.model_type == 'denoising': assert hparams.type_measurements == 'identity' if hparams.image_mode == '3D': out_img_list = [] for i in range(x_real.shape[-1]): # measurements and observation y_real = np.matmul(A, x_real[:, :, i].reshape( -1, 1)) + observ_noise # [n,n] * [n,1] -> [n,1] # tv norm f1 = functions.norm_tv(dim=1) # epsilon N = math.sqrt(sig_shape) epsilon = N * hparams.noise_level # L2 ball y = np.reshape(y_real, -1) #[n,1] -> [n,] f = functions.proj_b2(y=y, epsilon=epsilon) f2 = functions.func() # Indicator functions f2._eval = lambda x: 0 def prox(x, step): return np.reshape(f.prox(np.reshape(x, -1), 0), y_real.shape) f2._prox = prox # solver solver = solvers.douglas_rachford(step=0.1) x0 = np.array(y_real) #[n,1] ret = solvers.solve([f1, f2], x0, solver) # output out_img_piece = ret['sol'].reshape( x_real.shape[0], x_real.shape[1]) #[178,218] out_img_list.append(out_img_piece) out_img = np.transpose(np.array(out_img_list), (1, 2, 0)) else: # measurements and observation y_real = np.matmul(A, x_real.reshape( -1, 1)) + observ_noise # [n,n] * [n,1] -> [n,1] # tv norm f1 = functions.norm_tv(dim=1) # epsilon N = math.sqrt(sig_shape) epsilon = N * hparams.noise_level # L2 ball y = np.reshape(y_real, -1) #[n,1] -> [n,] f = functions.proj_b2(y=y, epsilon=epsilon) f2 = functions.func() # Indicator functions f2._eval = lambda x: 0 def prox(x, step): return np.reshape(f.prox(np.reshape(x, -1), 0), y_real.shape) f2._prox = prox # solver solver = solvers.douglas_rachford(step=0.1) x0 = np.array(y_real) #[n,1] ret = solvers.solve([f1, f2], x0, solver) # output out_img = ret['sol'] #[n,1] elif hparams.model_type == 'compressing': assert hparams.type_measurements == 'circulant' if hparams.image_mode == '3D': out_img_list = [] for i in range(x_real.shape[-1]): # construct observation g = circulant_np y_real = g(x_real[:, :, i].reshape(-1, 1)) #[n,1] -> [n,1] # tv norm f1 = functions.norm_tv(dim=1) # L2 norm tau = hparams.tau f2 = functions.norm_l2(y=y_real, A=g, lambda_=tau) # optimisation solver A_real = np.random.normal( size=(int(hparams.num_measurements / 3), sig_shape)) step = 0.5 / np.linalg.norm(A_real, ord=2)**2 solver = solvers.forward_backward( step=step ) #solver = solvers.forward_backward(step=0.5/tau) # initialisation x0 = np.array(y_real) #[n,1] # output ret = solvers.solve([f1, f2], x0, solver, rtol=1e-4, maxit=3000) #output = ret['sol'] out_img_piece = ret['sol'].reshape( x_real.shape[0], x_real.shape[1]) #[178,218] out_img_list.append(out_img_piece) out_img = np.transpose(np.array(out_img_list), (1, 2, 0)) else: # construct observation g = circulant_np y_real = g(x_real.reshape(-1, 1)) #[n,1] -> [n,1] # tv norm f1 = functions.norm_tv(dim=1) # L2 norm tau = hparams.tau f2 = functions.norm_l2(y=y_real, A=g, lambda_=tau) # optimisation solver A_real = np.random.normal(size=(hparams.num_measurements, sig_shape)) step = 0.5 / np.linalg.norm(A_real, ord=2)**2 solver = solvers.forward_backward( step=step ) #solver = solvers.forward_backward(step=0.5/tau) # initialisation x0 = np.array(y_real) #[n,1] # output ret = solvers.solve([f1, f2], x0, solver, rtol=1e-4, maxit=3000) #output = ret['sol'] out_img = ret['sol'] #[n,1] # ## === Lasso wavelet === ## elif hparams.decoder_type == 'lasso_wavelet': # Construct lasso wavelet functions def solve_lasso(A_val, y_val, hparams): #(n,m), (1,m) if hparams.lasso_solver == 'sklearn': lasso_est = Lasso(alpha=hparams.lmbd) lasso_est.fit(A_val.T, y_val.reshape(hparams.num_measurements)) x_hat = lasso_est.coef_ x_hat = np.reshape(x_hat, [-1]) elif hparams.lasso_solver == 'cvxopt': A_mat = matrix(A_val.T) #[m,n] y_mat = matrix(y_val.T) ### x_hat_mat = l1regls(A_mat, y_mat) x_hat = np.asarray(x_hat_mat) x_hat = np.reshape(x_hat, [-1]) #[n, ] elif hparams.lasso_solver == 'pyunlocbox': tau = hparams.tau f1 = functions.norm_l1(lambda_=tau) f2 = functions.norm_l2(y=y_val.T, A=A_val.T) if hparams.model_type == 'compressing': if hparams.image_mode == '3D': A_real = np.random.normal( size=(int(hparams.num_measurements / 3), sig_shape)) else: A_real = np.random.normal( size=(hparams.num_measurements, sig_shape)) step = 0.5 / np.linalg.norm(A_real, ord=2)**2 else: step = 0.5 / np.linalg.norm(A_val, ord=2)**2 solver = solvers.forward_backward(step=step) x0 = np.zeros((sig_shape, 1)) ret = solvers.solve([f1, f2], x0, solver, rtol=1e-4, maxit=3000) x_hat_mat = ret['sol'] x_hat = np.asarray(x_hat_mat) x_hat = np.reshape(x_hat, [-1]) #[n, ] return x_hat #generate basis def generate_basis(size): """generate the basis""" x = np.zeros((size, size)) coefs = pywt.wavedec2(x, 'db1') n_levels = len(coefs) basis = [] for i in range(n_levels): coefs[i] = list(coefs[i]) n_filters = len(coefs[i]) for j in range(n_filters): for m in range(coefs[i][j].shape[0]): try: for n in range(coefs[i][j].shape[1]): coefs[i][j][m][n] = 1 temp_basis = pywt.waverec2(coefs, 'db1') basis.append(temp_basis) coefs[i][j][m][n] = 0 except IndexError: coefs[i][j][m] = 1 temp_basis = pywt.waverec2(coefs, 'db1') basis.append(temp_basis) coefs[i][j][m] = 0 basis = np.array(basis) return basis def wavelet_basis(path_): if path_ == 'Ieeg_signal': W_ = generate_basis(32) W_ = W_.reshape((1024, 1024)) elif path_ == 'Celeb_signal': W_ = generate_basis(128) W_ = W_.reshape((16384, 16384)) else: W_ = generate_basis(64) W_ = W_.reshape((4096, 4096)) return W_ def lasso_wavelet_estimator(A_val, y_val, hparams): #(n,m), (1,m) W = wavelet_basis(hparams.path) #[n,n] if not callable(A_val): WA = np.dot(W, A_val) #[n,n] * [n,m] = [n,m] else: WA = np.array([ A_val(W[i, :].reshape(-1, 1)).reshape(-1) for i in range(len(W)) ]) #[n,n] -> [n,n] z_hat = solve_lasso(WA, y_val, hparams) # [n, ] x_hat = np.dot(z_hat, W) #[n, ] * [n,n] = [n, ] x_hat_max = np.abs(x_hat).max() x_hat = x_hat / (1.0 * x_hat_max) return x_hat # Construct inpainting masks def get_A_inpaint(mask_p): mask = mask_p.reshape(1, -1) A = np.eye(np.prod(mask.shape)) * np.tile(mask, [np.prod(mask.shape), 1]) A = np.asarray([a for a in A if np.sum(a) != 0]) A = np.sqrt( sig_shape ) * A # Make sure that the norm of each row of A is sig_shape assert all(np.abs(np.sum(A**2, 1) - sig_shape) < 1e-6) return A.T # Perofrm reconstruction if hparams.model_type == 'inpainting': # measurements and observation A_val = get_A_inpaint(mask) #(n,m) if hparams.image_mode == '3D': out_img_list = [] for i in range(x_real.shape[-1]): y_real = np.matmul(x_real[:, :, i].reshape(1, -1), A_val) #(1,m) out_img_piece = lasso_wavelet_estimator( A_val, y_real, hparams) out_img_piece = out_img_piece.reshape( x_real.shape[0], x_real.shape[1]) out_img_list.append(out_img_piece) out_img = np.transpose(np.array(out_img_list), (1, 2, 0)) elif hparams.image_mode == '1D': y_real = np.matmul(x_real.reshape(1, -1), A_val) #(1,m) out_img = lasso_wavelet_estimator(A_val, y_real, hparams) out_img = out_img.reshape(-1, 1) elif hparams.model_type == 'denoising': assert hparams.type_measurements == 'identity' A_val = A #(n,n) if hparams.image_mode == '3D': out_img_list = [] for i in range(x_real.shape[-1]): y_real = x_real[:, :, i].reshape(1, -1) + observ_noise.T out_img_piece = lasso_wavelet_estimator( A_val, y_real, hparams) out_img_piece = out_img_piece.reshape( x_real.shape[0], x_real.shape[1]) out_img_list.append(out_img_piece) out_img = np.transpose(np.array(out_img_list), (1, 2, 0)) elif hparams.image_mode == '1D': y_real = np.matmul(x_real.reshape(1, -1), A_val) + observ_noise.T out_img = lasso_wavelet_estimator(A_val, y_real, hparams) out_img = out_img.reshape(-1, 1) elif hparams.model_type == 'compressing': assert hparams.type_measurements == 'circulant' A_val = circulant_np if hparams.image_mode == '3D': out_img_list = [] for i in range(x_real.shape[-1]): y_real = A_val(x_real[:, :, i].reshape(-1, 1)).reshape( 1, -1) #[n,1] -> [1,n] out_img_piece = lasso_wavelet_estimator( A_val, y_real, hparams) out_img_piece = out_img_piece.reshape( x_real.shape[0], x_real.shape[1]) out_img_list.append(out_img_piece) out_img = np.transpose(np.array(out_img_list), (1, 2, 0)) elif hparams.image_mode == '1D': y_real = A_val(x_real).reshape(1, -1) #[n,1] -> [1,n] out_img = lasso_wavelet_estimator(A_val, y_real, hparams) out_img = out_img.reshape(-1, 1) ## === Printer === ## # Compute and print measurement and l2 loss # if hparams.image_mode == '3D' and hparams.model_type != 'inpainting': # x_real = x_real.reshape(-1,1) l2_losses = get_l2_loss(out_img, x_real, hparams.image_mode, hparams.decoder_type) psnr = 10 * np.log10(1 * 1 / l2_losses) #PSNR # Printer info if hparams.model_type == 'inpainting': if hparams.image_mode == '1D': mask_info = hparams.mask_name_1D[8:-4] elif hparams.image_mode == '2D' or hparams.image_mode == '3D': mask_info = hparams.mask_name_2D[8:-4] type_mea_info = 'NA' num_mea_info = 'NA' noise_level_info = 'NA' elif hparams.model_type == 'compressing': mask_info = 'NA' type_mea_info = hparams.type_measurements num_mea_info = str(hparams.num_measurements) noise_level_info = 'NA' elif hparams.model_type == 'denoising': mask_info = 'NA' type_mea_info = 'NA' num_mea_info = 'NA' noise_level_info = str(hparams.noise_level) # Print result print( 'Final representation PSNR for img_name:{}, model_type:{}, type_mea:{}, num_mea:{}, mask:{}, decoder:{} tau:{} noise:{} is {}' .format(hparams.img_name, hparams.model_type, type_mea_info, num_mea_info, mask_info, hparams.decoder_type, hparams.tau, noise_level_info, psnr)) print('END') print('\t') #sys.stdout.close() ## == to pd frame == ## if hparams.pickle == 1: pickle_file_path = hparams.pickle_file_path if not os.path.exists(pickle_file_path): d = { 'img_name': [hparams.img_name], 'model_type': [hparams.model_type], 'type_mea': [type_mea_info], 'num_mea': [num_mea_info], 'mask_info': [mask_info], 'decoder_type': [hparams.decoder_type], 'tau': [hparams.tau], 'noise': [noise_level_info], 'psnr': [psnr] } df = pd.DataFrame(data=d) df.to_pickle(pickle_file_path) else: d = { 'img_name': hparams.img_name, 'model_type': hparams.model_type, 'type_mea': type_mea_info, 'num_mea': num_mea_info, 'mask_info': mask_info, 'decoder_type': hparams.decoder_type, 'tau': hparams.tau, 'noise': noise_level_info, 'psnr': psnr } df = pd.read_pickle(pickle_file_path) df = df.append(d, ignore_index=True) df.to_pickle(pickle_file_path) ## === Save === ## if hparams.save == 1: save_out_img(out_img, 'result/', hparams.img_name, hparams.decoder_type, hparams.model_type, num_mea_info, mask_info, noise_level_info, hparams.image_mode)
def test_forward_backward_fista(self): """ Test forward-backward splitting solver with fista acceleration, solving problems with L1-norm, L2-norm, and dummy functions. """ y = [4., 5., 6., 7.] solver = solvers.forward_backward(accel=acceleration.fista()) param = {'solver': solver, 'rtol': 1e-6, 'verbosity': 'NONE'} # L2-norm prox and dummy gradient. f1 = functions.norm_l2(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 60) # Dummy prox and L2-norm gradient. f1 = functions.dummy() f2 = functions.norm_l2(y=y, lambda_=0.6) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 84) # L2-norm prox and L2-norm gradient. f1 = functions.norm_l2(y=y) f2 = functions.norm_l2(y=y) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y, rtol=1e-2) self.assertEqual(ret['crit'], 'MAXIT') self.assertEqual(ret['niter'], 200) # L1-norm prox and dummy gradient. f1 = functions.norm_l1(y=y) f2 = functions.dummy() ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 6) # Dummy prox and L1-norm gradient. As L1-norm possesses no gradient, # the algorithm exchanges the functions : exact same solution. f1 = functions.dummy() f2 = functions.norm_l1(y=y) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 6) # L1-norm prox and L1-norm gradient. L1-norm possesses no gradient. f1 = functions.norm_l1(y=y) f2 = functions.norm_l1(y=y) self.assertRaises(ValueError, solvers.solve, [f1, f2], np.zeros(len(y)), **param) # L1-norm prox and L2-norm gradient. f1 = functions.norm_l1(y=y, lambda_=1.0) f2 = functions.norm_l2(y=y, lambda_=0.8) ret = solvers.solve([f1, f2], np.zeros(len(y)), **param) nptest.assert_allclose(ret['sol'], y) self.assertEqual(ret['crit'], 'RTOL') self.assertEqual(ret['niter'], 10)
def test_mlfbf(self): """ Test the MLFBF solver with arbitrarily selected functions. """ x = [1., 1., 1.] L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]]) max_step = 1 / (1 + np.linalg.norm(L, 2)) solver = solvers.mlfbf(L=L, step=max_step / 2.) params = {'solver': solver, 'verbosity': 'NONE'} def x0(): return np.zeros(len(x)) # L2-norm prox and dummy prox. f = functions.dummy() f._prox = lambda x, T: np.maximum(np.zeros(len(x)), x) g = functions.norm_l2(lambda_=0.5) h = functions.norm_l2(y=np.array([294, 390, 361]), lambda_=0.5) ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params) nptest.assert_allclose(ret['sol'], x, rtol=1e-5) # Same test, but with callable L solver = solvers.mlfbf(L=lambda x: np.dot(L, x), Lt=lambda y: np.dot(L.T, y), d0=np.dot(L, x0()), step=max_step / 2.) ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params) nptest.assert_allclose(ret['sol'], x, rtol=1e-5) # Sanity check self.assertRaises(ValueError, solver.pre, [f, g], x0()) # Make a second test where the solution is calculated by hand n = 10 y = np.random.rand(n) * 2 z = np.random.rand(n) c = 1 delta = (y - z - c)**2 + 4 * (1 + y * z - z * c) sol = 0.5 * ((y - z - c) + np.sqrt(delta)) class mlog(functions.func): def __init__(self, z): super().__init__() self.z = z def _eval(self, x): return -np.sum(np.log(x + self.z)) def _prox(self, x, T): delta = (x - self.z)**2 + 4 * (T + x * self.z) sol = 0.5 * (x - self.z + np.sqrt(delta)) return sol f = functions.norm_l1(lambda_=c) g = mlog(z=z) h = functions.norm_l2(lambda_=0.5, y=y) mu = 1 + 1 step = 1 / mu / 2 solver = solvers.mlfbf(step=step) ret = solvers.solve([f, g, h], y.copy(), solver, maxit=200, rtol=0, verbosity="NONE") nptest.assert_allclose(ret["sol"], sol, atol=1e-10) # Make a final test where the function g can not be evaluate # on the primal variables y = np.random.rand(3) y_2 = L.dot(y) L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]]) x0 = np.zeros(len(y)) f = functions.norm_l1(y=y) g = functions.norm_l2(lambda_=0.5, y=y_2) h = functions.norm_l2(y=y, lambda_=0.5) max_step = 1 / (1 + np.linalg.norm(L, 2)) solver = solvers.mlfbf(L=L, step=max_step / 2.) ret = solvers.solve([f, g, h], x0, solver, maxit=1000, rtol=0) np.testing.assert_allclose(ret["sol"], y)
def graph_pnorm_interpolation(self, gradient, P, w, labels_bin, x0=None, p=1., **kwargs): r""" Solve an interpolation problem via gradient p-norm minimization. A signal :math:`x` is estimated from its measurements :math:`y = A(x)` by solving :math:`\text{arg}\underset{z \in \mathbb{R}^n}{\min} \| \nabla_G z \|_p^p \text{ subject to } Az = y` via a primal-dual, forward-backward-forward algorithm. Parameters ---------- gradient : array_like A matrix representing the graph gradient operator P : callable Orthogonal projection operator mapping points in :math:`z \in \mathbb{R}^n` onto the set satisfying :math:`A P(z) = A z`. x0 : array_like, optional Initial point of the iteration. Must be of dimension n. (Default is `numpy.random.randn(n)`) p : {1., 2.} labels_bin : array_like A vector that holds the binary labels. kwargs : Additional solver parameters, such as maximum number of iterations (maxit), relative tolerance on the objective (rtol), and verbosity level (verbosity). See :func:`pyunlocbox.solvers.solve` for the full list of options. Returns ------- x : array_like The solution to the optimization problem. """ grad = lambda z: gradient.dot(z) div = lambda z: gradient.transpose().dot(z) # Indicator function of the set satisfying :math:`y = A(z)` f = functions.func() f._eval = lambda z: 0 f._prox = lambda z, gamma: P(z, w, labels_bin) # :math:`\ell_1` norm of the dual variable :math:`d = \nabla_G z` g = functions.func() g._eval = lambda z: np.sum(np.abs(grad(z))) g._prox = lambda d, gamma: functions._soft_threshold(d, gamma) # :math:`\ell_2` norm of the gradient (for the smooth case) h = functions.norm_l2(A=grad, At=div) stepsize = (0.9 / (1. + scipy.sparse.linalg.norm(gradient, ord='fro'))) ** p solver = solvers.mlfbf(L=grad, Lt=div, step=stepsize) if p == 1.: problem = solvers.solve([f, g, functions.dummy()], x0=x0, solver=solver, **kwargs) return problem['sol'] if p == 2.: problem = solvers.solve([f, functions.dummy(), h], x0=x0, solver=solver, **kwargs) return problem['sol'] else: return x0
g = lambda x: mask * x im_masked = g(im_original) mask = 1 g = lambda x: mask * x from pyunlocbox import functions f1 = functions.norm_tv(maxit=50, dim=2) tau = 100 f2 = functions.norm_l2(y=im_masked, A=g, lambda_=tau) from pyunlocbox import solvers solver = solvers.forward_backward(step=0.5 / tau) x0 = np.array(im_masked) # Make a copy to preserve im_masked. ret = solvers.solve([f1, f2], x0, solver, maxit=100) import matplotlib.pyplot as plt fig = plt.figure(figsize=(8, 2.5)) ax1 = fig.add_subplot(1, 3, 1) _ = ax1.imshow(im_original, cmap='gray') _ = ax1.axis('off') _ = ax1.set_title('Original image') ax2 = fig.add_subplot(1, 3, 2) _ = ax2.imshow(im_masked, cmap='gray') _ = ax2.axis('off') _ = ax2.set_title('Masked image') ax3 = fig.add_subplot(1, 3, 3) _ = ax3.imshow(ret['sol'], cmap='gray') _ = ax3.axis('off') _ = ax3.set_title('Reconstructed image')
def run(scale=1.99, sigma_blur=0.1, noise_level_denoiser=0.005, num=None, method='FBS', pretrained_weights=True): if not os.path.isdir('results_conv'): os.mkdir('results_conv') # declare model act = tf.keras.activations.relu num_filters = 64 max_dim = 128 num_layers = 8 sizes = [None] * (num_layers) conv_shapes = [(num_filters, max_dim)] * num_layers filter_length = 5 model = StiefelModel(sizes, None, convolutional=True, filter_length=filter_length, dim=2, conv_shapes=conv_shapes, activation=act, scale_layer=scale) pred = model(tf.random.normal((10, 40, 40))) model.fast_execution = True # load weights if pretrained_weights: file_name = 'data/pretrained_weights/scale' + str( scale) + '_noise_level' + str(noise_level_denoiser) + '.pickle' else: if num is None: file_name = 'results_conv/scale' + str( scale) + '_noise_level' + str( noise_level_denoiser) + '/adam.pickle' else: file_name = 'results_conv/scale' + str( scale) + '_noise_level' + str( noise_level_denoiser) + '/adam' + str(num) + '.pickle' with open(file_name, 'rb') as f: trainable_vars = pickle.load(f) for i in range(len(model.trainable_variables)): model.trainable_variables[i].assign(trainable_vars[i]) beta = 1e8 project = True if project: # project convolution matrices on the Stiefel manifold for i in range(len(model.stiefel)): convs = model.stiefel[i].convs smaller = convs.shape[0] < convs.shape[1] if smaller: convs = transpose_convs(convs) iden = np.zeros((convs.shape[1], convs.shape[1], 4 * filter_length + 1, 4 * filter_length + 1), dtype=np.float32) for j in range(convs.shape[1]): iden[j, j, 2 * filter_length, 2 * filter_length] = 1 iden = tf.constant(iden) C = tf.identity(convs) def projection_objective(C): return 0.5 * beta * tf.reduce_sum( (conv_mult(transpose_convs(C), C) - iden)** 2) + .5 * tf.reduce_sum((C - convs)**2) for iteration in range(100): with tf.GradientTape(persistent=True) as tape: tape.watch(C) val = projection_objective(C) grad = tape.gradient(val, C) grad_sum = tf.reduce_sum(grad * grad) hess = tape.gradient(grad_sum, C) hess *= 0.5 / tf.sqrt(grad_sum) C -= grad / tf.sqrt(tf.reduce_sum(hess * hess)) if smaller: C = transpose_convs(C) model.stiefel[i].convs.assign(C) # load data test_directory = 'data/BSD68' fileList = os.listdir(test_directory + '/') fileList.sort() img_names = fileList save_path = 'results_conv/PnP_blur_' + method + str(sigma_blur) if not os.path.isdir(save_path): os.mkdir(save_path) if not os.path.isdir(save_path + '/blurred_data'): os.mkdir(save_path + '/blurred_data') if not os.path.isdir(save_path + '/l2tv'): os.mkdir(save_path + '/l2tv') psnr_sum = 0. psnr_noisy_sum = 0. psnr_l2tv_sum = 0. error_sum = 0. error_bm3d_sum = 0. counter = 0 sig = sigma_blur sig_sq = sig**2 noise_level = 0.01 kernel_width = 9 x_range = 1. * np.array(range(kernel_width)) kernel_x = np.tile(x_range[:, np.newaxis], (1, kernel_width)) - .5 * (kernel_width - 1) y_range = 1. * np.array(range(kernel_width)) kernel_y = np.tile(y_range[np.newaxis, :], (kernel_width, 1)) - .5 * (kernel_width - 1) kernel = np.exp(-(kernel_x**2 + kernel_y**2) / (2 * sig_sq)) kernel /= np.sum(kernel) kernel = tf.constant(kernel, dtype=tf.float32) myfile = open(save_path + "/psnrs.txt", "w") myfile.write("PSNRs:\n") myfile.close() np.random.seed(25) for name in img_names: # load image and compute blurred version counter += 1 img = Image.open(test_directory + '/' + name) img = img.convert('L') img_gray = 1.0 * np.array(img) img_gray /= 255.0 img_gray_pil = Image.fromarray(img_gray * 255.0) img_gray_pil = img_gray_pil.convert('RGB') img_gray_pil.save(save_path + '/original' + name) one_img = tf.ones(img_gray.shape) img_blurred = tf.nn.conv2d( tf.expand_dims( tf.expand_dims(tf.constant(img_gray, dtype=tf.float32), 0), -1), tf.expand_dims(tf.expand_dims(kernel, -1), -1), 1, 'SAME') img_blurred = tf.squeeze(img_blurred).numpy() ones_blurred = tf.nn.conv2d( tf.expand_dims( tf.expand_dims(tf.constant(one_img, dtype=tf.float32), 0), -1), tf.expand_dims(tf.expand_dims(kernel, -1), -1), 1, 'SAME') ones_blurred = tf.squeeze(ones_blurred).numpy() img_blurred /= ones_blurred noise = np.random.normal(0, 1, img_blurred.shape) img_blurred += noise_level * noise pad = kernel_width // 2 img_obs = img_blurred[pad:-pad, pad:-pad] img_start = np.pad(img_obs, ((pad, pad), (pad, pad)), 'edge') img_obs_big = np.concatenate([ np.zeros((img_obs.shape[0], pad)), img_obs, np.zeros((img_obs.shape[0], pad)) ], 1) img_obs_big = np.concatenate([ np.zeros((pad, img_obs_big.shape[1])), img_obs_big, np.zeros((pad, img_obs_big.shape[1])) ], 0) savemat(save_path + '/blurred_data/' + name[:-4] + '_blurred.mat', {'img_blur': (img_blurred) * 255}) scalar = scale alpha_star = 0.5 conv_coord = 1 - scalar + 2 * alpha_star * scalar # declare functions for PnP def my_f(signal, inp_signal): signal_blurred = tf.nn.conv2d( tf.expand_dims(signal, -1), tf.expand_dims(tf.expand_dims(kernel, -1), -1), 1, 'VALID') signal_blurred = tf.reshape(signal_blurred, signal_blurred.shape[:3]) out = .5 * tf.reduce_sum((signal_blurred - img_obs)**2) return out def prox_my_f(signal, lam, inp_signal): out_signal = tf.identity(signal) for i in range(50): with tf.GradientTape(persistent=True) as tape: tape.watch(out_signal) term1 = my_f(out_signal, inp_signal) term2 = .5 * tf.reduce_sum((out_signal - signal)**2) objective = term1 / lam + term2 grad = tape.gradient(objective, out_signal) grad_sum = tf.reduce_sum(grad**2) hess = .5 * tape.gradient(grad_sum, out_signal) / tf.sqrt(grad_sum) out_signal -= grad / tf.sqrt(tf.reduce_sum(hess**2)) return out_signal def grad_f(signal): signal_blurred = tf.nn.conv2d( tf.expand_dims(signal, -1), tf.expand_dims(tf.expand_dims(kernel, -1), -1), 1, 'SAME') signal_blurred_minus_inp = tf.reshape( signal_blurred, signal_blurred.shape[:3]) - img_blurred AtA = tf.nn.conv2d(tf.expand_dims(signal_blurred_minus_inp, -1), tf.expand_dims(tf.expand_dims(kernel, -1), -1), 1, 'SAME') AtA = tf.reshape(AtA, signal_blurred.shape[:3]) return AtA #L2-TV def g(signal): signal_blurred = tf.nn.conv2d( tf.expand_dims( tf.expand_dims(tf.constant(signal, tf.float32), -1), 0), tf.expand_dims(tf.expand_dims(kernel, -1), -1), 1, 'VALID') signal_blurred = tf.squeeze(signal_blurred) signal_blurred = np.concatenate([ np.zeros((signal_blurred.shape[0], pad)), signal_blurred.numpy(), np.zeros((signal_blurred.shape[0], pad)) ], 1) signal_blurred = np.concatenate([ np.zeros((pad, signal_blurred.shape[1])), signal_blurred, np.zeros((pad, signal_blurred.shape[1])) ], 0) return signal_blurred f1 = functions.norm_tv(maxit=50, dim=2) l2tv_lambda = 0.001 f2 = functions.norm_l2(y=img_obs_big, A=g, lambda_=1 / l2tv_lambda) solver = solvers.forward_backward(step=0.5 * l2tv_lambda) img_blurred2 = tf.identity(img_start).numpy() l2tv = solvers.solve([f1, f2], img_blurred2, solver, maxit=100, verbosity='NONE') l2tv = l2tv['sol'] def my_T(inp, model): my_fac = 1. return (1 - 1 / (conv_coord)) * l2tv + 1 / (conv_coord) * (inp - model( (inp - .5) * my_fac)) # Compute PnP result if method == 'FBS': pred = PnP_FBS(model, l2tv[np.newaxis, :, :], tau=1.9, T_fun=my_T, eps=1e-3, fun=my_f) elif method == 'ADMM': pred = PnP_ADMM(l2tv[np.newaxis, :, :], lambda x: my_T(x, model), gamma=.52, prox_fun=prox_my_f) else: raise ValueError('Unknown method!') # save results noisy = (img_start) * 255 reconstructed = (tf.reshape( pred, [pred.shape[1], pred.shape[2]]).numpy()) * 255. img_gray = (img_gray) * 255. l2tv *= 255 error_sum += tf.reduce_sum( ((reconstructed - img_gray) / 255.)**2).numpy() psnr = meanPSNR( tf.keras.backend.flatten(reconstructed[2 * pad:-2 * pad, 2 * pad:-2 * pad]).numpy() / 255.0, tf.keras.backend.flatten( img_gray[2 * pad:-2 * pad, 2 * pad:-2 * pad]).numpy() / 255.0, one_dist=True) psnr_l2tv = meanPSNR( tf.keras.backend.flatten(l2tv[2 * pad:-2 * pad, 2 * pad:-2 * pad]).numpy() / 255.0, tf.keras.backend.flatten( img_gray[2 * pad:-2 * pad, 2 * pad:-2 * pad]).numpy() / 255.0, one_dist=True) psnr_noisy = meanPSNR( tf.keras.backend.flatten(noisy[2 * pad:-2 * pad, 2 * pad:-2 * pad]).numpy() / 255.0, tf.keras.backend.flatten( img_gray[2 * pad:-2 * pad, 2 * pad:-2 * pad]).numpy() / 255.0, one_dist=True) print('PSNR of ' + name + ': ' + str(psnr)) print('PSNR L2TV of ' + name + ': ' + str(psnr_l2tv)) print('PSNR of noisy ' + name + ': ' + str(psnr_noisy)) psnr_sum += psnr psnr_noisy_sum += psnr_noisy psnr_l2tv_sum += psnr_l2tv print('Mean PSNR PPNN: ' + str(psnr_sum / counter)) print('Mean PSNR L2TV: ' + str(psnr_l2tv_sum / counter)) print('Mean PSNR noisy: ' + str(psnr_noisy_sum / counter)) myfile = open(save_path + "/psnrs.txt", "a") myfile.write('PSNR of ' + name + ': ' + str(psnr) + '\n') myfile.write('PSNR L2TV of ' + name + ': ' + str(psnr_l2tv) + '\n') myfile.write('PSNR of noisy ' + name + ': ' + str(psnr_noisy) + '\n') myfile.close() img = Image.fromarray(noisy) img = img.convert('RGB') img.save(save_path + '/noisy' + name) img = Image.fromarray(l2tv) img = img.convert('RGB') img.save(save_path + '/l2tv/l2tv' + name) img = Image.fromarray(reconstructed) img = img.convert('RGB') img.save(save_path + '/reconstructed' + name) print('Mean PSNR on images: ' + str(psnr_sum / len(img_names))) print('Mean PSNR on noisy images: ' + str(psnr_noisy_sum / len(img_names)))
def test_solve(self): """ Test some features of the solving function. """ # We have to set a seed here for the random draw if we are required # below to assert that the number of iterations of the solvers are # equal to some specific values. Otherwise, we get trivial errors when # x0 is a little farther away from y in a given draw. rs = np.random.RandomState(42) y = 5 - 10 * rs.uniform(size=(15, 4)) def x0(): return np.zeros(y.shape) nverb = {'verbosity': 'NONE'} # Function verbosity. f = functions.dummy() self.assertEqual(f.verbosity, 'NONE') f.verbosity = 'LOW' solvers.solve([f], x0(), **nverb) self.assertEqual(f.verbosity, 'LOW') # Input parameters. self.assertRaises(ValueError, solvers.solve, [f], x0(), verbosity='??') # Addition of dummy function. self.assertRaises(ValueError, solvers.solve, [], x0(), **nverb) solver = solvers.forward_backward() solvers.solve([f], x0(), solver, **nverb) # self.assertIsInstance(solver.f1, functions.dummy) # self.assertIsInstance(solver.f2, functions.dummy) # Automatic solver selection. f0 = functions.func() f0._eval = lambda x: 0 f0._grad = lambda x: x f1 = functions.func() f1._eval = lambda x: 0 f1._grad = lambda x: x f1._prox = lambda x, T: x f2 = functions.func() f2._eval = lambda x: 0 f2._prox = lambda x, T: x self.assertRaises(ValueError, solvers.solve, [f0, f0], x0(), **nverb) ret = solvers.solve([f0, f1], x0(), **nverb) self.assertEqual(ret['solver'], 'forward_backward') ret = solvers.solve([f1, f0], x0(), **nverb) self.assertEqual(ret['solver'], 'forward_backward') ret = solvers.solve([f1, f2], x0(), **nverb) self.assertEqual(ret['solver'], 'forward_backward') ret = solvers.solve([f2, f2], x0(), **nverb) self.assertEqual(ret['solver'], 'douglas_rachford') ret = solvers.solve([f1, f2, f0], x0(), **nverb) self.assertEqual(ret['solver'], 'generalized_forward_backward') # Stopping criteria. f = functions.norm_l2(y=y) tol = 1e-6 r = solvers.solve([f], x0(), None, tol, None, None, None, None, 'NONE') self.assertEqual(r['crit'], 'ATOL') self.assertLess(np.sum(r['objective'][-1]), tol) self.assertEqual(r['niter'], 9) tol = 1e-8 r = solvers.solve([f], x0(), None, None, tol, None, None, None, 'NONE') self.assertEqual(r['crit'], 'DTOL') err = np.abs(np.sum(r['objective'][-1]) - np.sum(r['objective'][-2])) self.assertLess(err, tol) self.assertEqual(r['niter'], 17) tol = .1 r = solvers.solve([f], x0(), None, None, None, tol, None, None, 'NONE') self.assertEqual(r['crit'], 'RTOL') err = np.abs(np.sum(r['objective'][-1]) - np.sum(r['objective'][-2])) err /= np.sum(r['objective'][-1]) self.assertLess(err, tol) self.assertEqual(r['niter'], 13) tol = 1e-4 r = solvers.solve([f], x0(), None, None, None, None, tol, None, 'NONE') self.assertEqual(r['crit'], 'XTOL') r2 = solvers.solve([f], x0(), maxit=r['niter'] - 1, **nverb) err = np.linalg.norm(r['sol'] - r2['sol']) / np.sqrt(x0().size) self.assertLess(err, tol) self.assertEqual(r['niter'], 14) nit = 15 r = solvers.solve([f], x0(), None, None, None, None, None, nit, 'NONE') self.assertEqual(r['crit'], 'MAXIT') self.assertEqual(r['niter'], nit) # Return values. f = functions.norm_l2(y=y) ret = solvers.solve([f], x0(), **nverb) self.assertEqual(len(ret), 6) self.assertIsInstance(ret['sol'], np.ndarray) self.assertIsInstance(ret['solver'], str) self.assertIsInstance(ret['crit'], str) self.assertIsInstance(ret['niter'], int) self.assertIsInstance(ret['time'], float) self.assertIsInstance(ret['objective'], list)
plt.figure(14) plt.title("Estimated Super resolution gaussian - Wiener filter") plt.imshow(H_estimated_wf_gaussian, cmap='gray') plt.figure(15) plt.title("Estimated Super resolution box - Wiener filter") plt.imshow(H_estimated_wf_box, cmap='gray') # Task 5.2 tau = 100 g = lambda H: signal.convolve2d(H, K_gaussian, boundary='symm', mode='same') l_blurred_cpy = np.array(blurred_image_gaussian_l) tv_prior_f = functions.norm_tv(maxit=50, dim=2) norm_l2_f = functions.norm_l2(y=l_blurred_cpy, A=g, lambda_=tau) solver = solvers.forward_backward(step=0.0001 / tau) H_estimated_lms_tv_gaussian = solvers.solve([tv_prior_f, norm_l2_f], l_blurred_cpy, solver, maxit=100) g = lambda H: signal.convolve2d(H, K_box, boundary='symm', mode='same') l_blurred_cpy = np.array(blurred_image_box_l) tv_prior_f = functions.norm_tv(maxit=50, dim=2) norm_l2_f = functions.norm_l2(y=l_blurred_cpy, A=g, lambda_=tau) solver = solvers.forward_backward(step=0.0001 / tau) H_estimated_lms_tv_box = solvers.solve([tv_prior_f, norm_l2_f], l_blurred_cpy, solver, maxit=100) plt.figure(16) plt.title("Estimated Super resolution gaussian - Least mean square with TV prior") plt.imshow(H_estimated_lms_tv_gaussian['sol'], cmap='gray') plt.figure(17) plt.title("Estimated Super resolution box - Least mean square with TV prior") plt.imshow(H_estimated_lms_tv_box['sol'], cmap='gray')
def log_degree_barrier(X, dist_type='sqeuclidean', alpha=1, beta=1, step=0.5, w0=None, maxit=1000, rtol=1e-5, retall=False, verbosity='NONE'): r""" Learn graph by imposing a log barrier on the degrees This is done by solving :math:`\tilde{W} = \underset{W \in \mathcal{W}_m}{\text{arg}\min} \, \|W \odot Z\|_{1,1} - \alpha 1^{T} \log{W1} + \beta \| W \|_{F}^{2}`, where :math:`Z` is a pairwise distance matrix, and :math:`\mathcal{W}_m` is the set of valid symmetric weighted adjacency matrices. Parameters ---------- X : array_like An N-by-M data matrix of N variable observations in an M-dimensional space. The learned graph will have N nodes. dist_type : string Type of pairwise distance between variables. See :func:`spatial.distance.pdist` for the possible options. alpha : float, optional Regularization parameter acting on the log barrier beta : float, optional Regularization parameter controlling the density of the graph step : float, optional A number between 0 and 1 defining a stepsize value in the admissible stepsize interval (see [Komodakis & Pesquet, 2015], Algorithm 6) w0 : array_like, optional Initialization of the edge weights. Must be an N(N-1)/2-dimensional vector. maxit : int, optional Maximum number of iterations. rtol : float, optional Stopping criterion. Relative tolerance between successive updates. retall : boolean Return solution and problem details. See output of :func:`pyunlocbox.solvers.solve`. verbosity : {'NONE', 'LOW', 'HIGH', 'ALL'}, optional Level of verbosity of the solver. See :func:`pyunlocbox.solvers.solve`. Returns ------- W : array_like Learned weighted adjacency matrix problem : dict, optional Information about the solution of the optimization. Only returned if retall == True. Notes ----- This is the solver proposed in [Kalofolias, 2016] :cite:`kalofolias2016`. Examples -------- >>> import learn_graph as lg >>> import networkx as nx >>> import numpy as np >>> import matplotlib.pyplot as plt >>> from scipy import spatial >>> G_gt = nx.waxman_graph(100) >>> pos = nx.random_layout(G_gt) >>> coords = np.array(list(pos.values())) >>> def s1(x, y): return np.sin((2 - x - y)**2) >>> def s2(x, y): return np.cos((x + y)**2) >>> def s3(x, y): return (x - 0.5)**2 + (y - 0.5)**3 + x - y >>> def s4(x, y): return np.sin(3 * ( (x - 0.5)**2 + (y - 0.5)**2 ) ) >>> X = np.array((s1(coords[:,0], coords[:,1]), s2(coords[:,0], coords[:,1]), s3(coords[:,0], coords[:,1]), s4(coords[:,0], coords[:,1]))).T >>> z = 25 * spatial.distance.pdist(X, 'sqeuclidean') >>> W = lg.log_degree_barrier(z) >>> W[W < np.percentile(W, 96)] = 0 >>> G_learned = nx.from_numpy_matrix(W) >>> plt.figure(figsize=(12, 6)) >>> plt.subplot(1,2,1) >>> nx.draw(G_gt, pos=pos) >>> plt.title('Ground Truth') >>> plt.subplot(1,2,2) >>> nx.draw(G_learned, pos=pos) >>> plt.title('Learned') """ # Parse X N = X.shape[0] z = spatial.distance.pdist(X, dist_type) # Pairwise distances # Parse stepsize if (step <= 0) or (step > 1): raise ValueError("step must be a number between 0 and 1.") # Parse initial weights w0 = np.zeros(z.shape) if w0 is None else w0 if (w0.shape != z.shape): raise ValueError("w0 must be of dimension N(N-1)/2.") # Get primal-dual linear map K, Kt = utils.weight2degmap(N) norm_K = np.sqrt(2 * (N - 1)) # Assemble functions in the objective f1 = functions.func() f1._eval = lambda w: 2 * np.dot(w, z) f1._prox = lambda w, gamma: np.maximum(0, w - (2 * gamma * z)) f2 = functions.func() f2._eval = lambda w: -alpha * np.sum( np.log(np.maximum(np.finfo(np.float64).eps, K(w)))) f2._prox = lambda d, gamma: np.maximum( 0, 0.5 * (d + np.sqrt(d**2 + (4 * alpha * gamma)))) f3 = functions.func() f3._eval = lambda w: beta * np.sum(w**2) f3._grad = lambda w: 2 * beta * w lipg = 2 * beta # Rescale stepsize stepsize = step / (1 + lipg + norm_K) # Solve problem solver = solvers.mlfbf(L=K, Lt=Kt, step=stepsize) problem = solvers.solve([f1, f2, f3], x0=w0, solver=solver, maxit=maxit, rtol=rtol, verbosity=verbosity) # Transform weight matrix from vector form to matrix form W = spatial.distance.squareform(problem['sol']) if retall: return W, problem else: return W
def l2_degree_reg(X, dist_type='sqeuclidean', alpha=1, s=None, step=0.5, w0=None, maxit=1000, rtol=1e-5, retall=False, verbosity='NONE'): r""" Learn graph by regularizing the l2-norm of the degrees. This is done by solving :math:`\tilde{W} = \underset{W \in \mathcal{W}_m}{\text{arg}\min} \, \|W \odot Z\|_{1,1} + \alpha \|W1}\|^2 + \alpha \| W \|_{F}^{2}`, subject to :math:`\|W\|_{1,1} = s`, where :math:`Z` is a pairwise distance matrix, and :math:`\mathcal{W}_m`is the set of valid symmetric weighted adjacency matrices. Parameters ---------- X : array_like An N-by-M data matrix of N variable observations in an M-dimensional space. The learned graph will have N nodes. dist_type : string Type of pairwise distance between variables. See :func:`spatial.distance.pdist` for the possible options. alpha : float, optional Regularization parameter acting on the l2-norm. s : float, optional The "sparsity level" of the weight matrix, as measured by its l1-norm. step : float, optional A number between 0 and 1 defining a stepsize value in the admissible stepsize interval (see [Komodakis & Pesquet, 2015], Algorithm 6) w0 : array_like, optional Initialization of the edge weights. Must be an N(N-1)/2-dimensional vector. maxit : int, optional Maximum number of iterations. rtol : float, optional Stopping criterion. Relative tolerance between successive updates. retall : boolean Return solution and problem details. See output of :func:`pyunlocbox.solvers.solve`. verbosity : {'NONE', 'LOW', 'HIGH', 'ALL'}, optional Level of verbosity of the solver. See :func:`pyunlocbox.solvers.solve`. Returns ------- W : array_like Learned weighted adjacency matrix problem : dict, optional Information about the solution of the optimization. Only returned if retall == True. Notes ----- This is the problem proposed in [Dong et al., 2015]. Examples -------- """ # Parse X N = X.shape[0] E = int(N * (N - 1.) / 2.) z = spatial.distance.pdist(X, dist_type) # Pairwise distances # Parse s s = N if s is None else s # Parse step if (step <= 0) or (step > 1): raise ValueError("step must be a number between 0 and 1.") # Parse initial weights w0 = np.zeros(z.shape) if w0 is None else w0 if (w0.shape != z.shape): raise ValueError("w0 must be of dimension N(N-1)/2.") # Get primal-dual linear map one_vec = np.ones(E) def K(w): return np.array([2 * np.dot(one_vec, w)]) def Kt(n): return 2 * n * one_vec norm_K = 2 * np.sqrt(E) # Get weight-to-degree map S, St = utils.weight2degmap(N) # Assemble functions in the objective f1 = functions.func() f1._eval = lambda w: 2 * np.dot(w, z) f1._prox = lambda w, gamma: np.maximum(0, w - (2 * gamma * z)) f2 = functions.func() f2._eval = lambda w: 0. f2._prox = lambda d, gamma: s f3 = functions.func() f3._eval = lambda w: alpha * (2 * np.sum(w**2) + np.sum(S(w)**2)) f3._grad = lambda w: alpha * (4 * w + St(S(w))) lipg = 2 * alpha * (N + 1) # Rescale stepsize stepsize = step / (1 + lipg + norm_K) # Solve problem solver = solvers.mlfbf(L=K, Lt=Kt, step=stepsize) problem = solvers.solve([f1, f2, f3], x0=w0, solver=solver, maxit=maxit, rtol=rtol, verbosity=verbosity) # Transform weight matrix from vector form to matrix form W = spatial.distance.squareform(problem['sol']) if retall: return W, problem else: return W
def classification_tikhonov_simplex(G, y, M, tau=0.1, **kwargs): r"""Solve a classification problem on graph via Tikhonov minimization with simple constraints. The function first transforms :math:`y` in logits :math:`Y`, then solves .. math:: \operatorname*{arg min}_X \| M X - Y \|_2^2 + \tau \ tr(X^T L X) \text{ s.t. } sum(X) = 1 \text{ and } X >= 0, where :math:`X` and :math:`Y` are logits. Parameters ---------- G : :class:`pygsp.graphs.Graph` y : array, length G.n_vertices Measurements. M : array of boolean, length G.n_vertices Masking vector. tau : float Regularization parameter. kwargs : dict Parameters for :func:`pyunlocbox.solvers.solve`. Returns ------- logits : array, length G.n_vertices The logits :math:`X`. Examples -------- >>> from pygsp import graphs, learning >>> import matplotlib.pyplot as plt >>> >>> G = graphs.Logo() >>> G.estimate_lmax() Create a ground truth signal: >>> signal = np.zeros(G.n_vertices) >>> signal[G.info['idx_s']] = 1 >>> signal[G.info['idx_p']] = 2 Construct a measurement signal from a binary mask: >>> rs = np.random.RandomState(42) >>> mask = rs.uniform(0, 1, G.n_vertices) > 0.5 >>> measures = signal.copy() >>> measures[~mask] = np.nan Solve the classification problem by reconstructing the signal: >>> recovery = learning.classification_tikhonov_simplex( ... G, measures, mask, tau=0.1, verbosity='NONE') Plot the results. Note that we recover the class with ``np.argmax(recovery, axis=1)``. >>> prediction = np.argmax(recovery, axis=1) >>> fig, ax = plt.subplots(2, 3, sharey=True, figsize=(10, 6)) >>> _ = G.plot_signal(signal, ax=ax[0, 0], title='Ground truth') >>> _ = G.plot_signal(measures, ax=ax[0, 1], title='Measurements') >>> _ = G.plot_signal(prediction, ax=ax[0, 2], title='Recovered class') >>> _ = G.plot_signal(recovery[:, 0], ax=ax[1, 0], title='Logit 0') >>> _ = G.plot_signal(recovery[:, 1], ax=ax[1, 1], title='Logit 1') >>> _ = G.plot_signal(recovery[:, 2], ax=ax[1, 2], title='Logit 2') >>> _ = fig.tight_layout() """ functions, solvers = _import_pyunlocbox() if tau <= 0: raise ValueError('Tau should be greater than 0.') y = y.copy() y[M == False] = 0 Y = _to_logits(y.astype(np.int)) Y[M == False, :] = 0 def proj_simplex(y): d = y.shape[1] a = np.ones(d) idx = np.argsort(y) def evalpL(y, k, idx): return np.sum(y[idx[k:]] - y[idx[k]]) - 1 def bisectsearch(idx, y): idxL, idxH = 0, d - 1 L = evalpL(y, idxL, idx) H = evalpL(y, idxH, idx) if L < 0: return idxL while (idxH - idxL) > 1: iMid = int((idxL + idxH) / 2) M = evalpL(y, iMid, idx) if M > 0: idxL, L = iMid, M else: idxH, H = iMid, M return idxH def proj(idx, y): k = bisectsearch(idx, y) lam = (np.sum(y[idx[k:]]) - 1) / (d - k) return np.maximum(0, y - lam) x = np.empty_like(y) for i in range(len(y)): x[i] = proj(idx[i], y[i]) # x = np.stack(map(proj, idx, y)) return x def smooth_eval(x): xTLx = np.sum(x * (G.L.dot(x))) e = M * ((M * x.T) - Y.T) l2 = np.sum(e * e) return tau * xTLx + l2 def smooth_grad(x): return 2 * ((M * (M * x.T - Y.T)).T + tau * G.L * x) f1 = functions.func() f1._eval = smooth_eval f1._grad = smooth_grad f2 = functions.func() f2._eval = lambda x: 0 # Indicator functions evaluate to zero. f2._prox = lambda x, step: proj_simplex(x) step = 0.5 / (1 + tau * G.lmax) solver = solvers.forward_backward(step=step) ret = solvers.solve([f1, f2], Y.copy(), solver, **kwargs) return ret['sol']
def classification_tikhonov_simplex(G, y, M, tau=0.1, **kwargs): r"""Solve a classification problem on graph via Tikhonov minimization with simple constraints. The function first transforms :math:`y` in logits :math:`Y`, then solves .. math:: \operatorname*{arg min}_X \| M X - Y \|_2^2 + \tau \ tr(X^T L X) \text{ s.t. } sum(X) = 1 \text{ and } X >= 0, where :math:`X` and :math:`Y` are logits. Parameters ---------- G : :class:`pygsp.graphs.Graph` y : array, length G.n_vertices Measurements. M : array of boolean, length G.n_vertices Masking vector. tau : float Regularization parameter. kwargs : dict Parameters for :func:`pyunlocbox.solvers.solve`. Returns ------- logits : array, length G.n_vertices The logits :math:`X`. Examples -------- >>> from pygsp import graphs, learning >>> import matplotlib.pyplot as plt >>> >>> G = graphs.Logo() >>> G.estimate_lmax() Create a ground truth signal: >>> signal = np.zeros(G.n_vertices) >>> signal[G.info['idx_s']] = 1 >>> signal[G.info['idx_p']] = 2 Construct a measurement signal from a binary mask: >>> rs = np.random.RandomState(42) >>> mask = rs.uniform(0, 1, G.n_vertices) > 0.5 >>> measures = signal.copy() >>> measures[~mask] = np.nan Solve the classification problem by reconstructing the signal: >>> recovery = learning.classification_tikhonov_simplex( ... G, measures, mask, tau=0.1, verbosity='NONE') Plot the results. Note that we recover the class with ``np.argmax(recovery, axis=1)``. >>> prediction = np.argmax(recovery, axis=1) >>> fig, ax = plt.subplots(2, 3, sharey=True, figsize=(10, 6)) >>> _ = G.plot_signal(signal, ax=ax[0, 0], title='Ground truth') >>> _ = G.plot_signal(measures, ax=ax[0, 1], title='Measurements') >>> _ = G.plot_signal(prediction, ax=ax[0, 2], title='Recovered class') >>> _ = G.plot_signal(recovery[:, 0], ax=ax[1, 0], title='Logit 0') >>> _ = G.plot_signal(recovery[:, 1], ax=ax[1, 1], title='Logit 1') >>> _ = G.plot_signal(recovery[:, 2], ax=ax[1, 2], title='Logit 2') >>> _ = fig.tight_layout() """ functions, solvers = _import_pyunlocbox() if tau <= 0: raise ValueError('Tau should be greater than 0.') y[M == False] = 0 Y = _to_logits(y.astype(np.int)) Y[M == False, :] = 0 def proj_simplex(y): d = y.shape[1] a = np.ones(d) idx = np.argsort(y) def evalpL(y, k, idx): return np.sum(y[idx[k:]] - y[idx[k]]) - 1 def bisectsearch(idx, y): idxL, idxH = 0, d-1 L = evalpL(y, idxL, idx) H = evalpL(y, idxH, idx) if L < 0: return idxL while (idxH-idxL) > 1: iMid = int((idxL + idxH) / 2) M = evalpL(y, iMid, idx) if M > 0: idxL, L = iMid, M else: idxH, H = iMid, M return idxH def proj(idx, y): k = bisectsearch(idx, y) lam = (np.sum(y[idx[k:]]) - 1) / (d - k) return np.maximum(0, y - lam) x = np.empty_like(y) for i in range(len(y)): x[i] = proj(idx[i], y[i]) # x = np.stack(map(proj, idx, y)) return x def smooth_eval(x): xTLx = np.sum(x * (G.L.dot(x))) e = M * ((M * x.T) - Y.T) l2 = np.sum(e * e) return tau * xTLx + l2 def smooth_grad(x): return 2 * ((M * (M * x.T - Y.T)).T + tau * G.L * x) f1 = functions.func() f1._eval = smooth_eval f1._grad = smooth_grad f2 = functions.func() f2._eval = lambda x: 0 # Indicator functions evaluate to zero. f2._prox = lambda x, step: proj_simplex(x) step = 0.5 / (1 + tau * G.lmax) solver = solvers.forward_backward(step=step) ret = solvers.solve([f1, f2], Y.copy(), solver, **kwargs) return ret['sol']