def test_nonoverlapping_nonsmooth(self): # Spams: http://spams-devel.gforge.inria.fr/doc-python/doc_spams.pdf import numpy as np from parsimony.functions import CombinedFunction import parsimony.algorithms.proximal as proximal import parsimony.functions as functions import parsimony.functions.nesterov.gl as gl import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl import parsimony.utils.start_vectors as start_vectors np.random.seed(42) # Note that p must be even! n, p = 25, 20 groups = [range(0, p / 2), range(p / 2, p)] # weights = [1.5, 0.5] A = gl.A_from_groups(p, groups=groups) # , weights=weights) l = 0.0 k = 0.0 g = 1.0 start_vector = start_vectors.RandomStartVector(normalise=True) beta = start_vector.get_vector(p) alpha = 1.0 Sigma = alpha * np.eye(p, p) \ + (1.0 - alpha) * np.random.randn(p, p) mean = np.zeros(p) M = np.random.multivariate_normal(mean, Sigma, n) e = np.random.randn(n, 1) snr = 100.0 X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr) eps = 1e-8 max_iter = 8500 beta_start = start_vector.get_vector(p) mus = [5e-2, 5e-4, 5e-6, 5e-8] fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus)) beta_parsimony = beta_start for mu in mus: # function = functions.LinearRegressionL1L2GL(X, y, l, k, g, # A=A, mu=mu, # penalty_start=0) function = CombinedFunction() function.add_function(functions.losses.LinearRegression(X, y, mean=False)) function.add_penalty(gl.GroupLassoOverlap(l=g, A=A, mu=mu, penalty_start=0)) beta_parsimony = fista.run(function, beta_parsimony) try: import spams params = {"loss": "square", "regul": "group-lasso-l2", "groups": np.array([1] * (p / 2) + [2] * (p / 2), dtype=np.int32), "lambda1": g, "max_it": max_iter, "tol": eps, "ista": False, "numThreads": -1, } beta_spams, optim_info = \ spams.fistaFlat(Y=np.asfortranarray(y), X=np.asfortranarray(X), W0=np.asfortranarray(beta_start), return_optim_info=True, **params) except ImportError: beta_spams = np.asarray([[14.01111427], [35.56508563], [27.38245962], [22.39716553], [5.835744940], [5.841502910], [2.172209350], [32.40227785], [22.48364756], [26.48822401], [0.770391500], [36.28288883], [31.14118214], [7.938279340], [6.800713150], [6.862914540], [11.38161678], [19.63087584], [16.15855845], [10.89356615]]) berr = np.linalg.norm(beta_parsimony - beta_spams) # print berr assert berr < 5e-2 f_parsimony = function.f(beta_parsimony) f_spams = function.f(beta_spams) ferr = abs(f_parsimony - f_spams) # print ferr assert ferr < 5e-6
def test_combo_overlapping_nonsmooth(self): import numpy as np from parsimony.functions import CombinedFunction import parsimony.algorithms.proximal as proximal import parsimony.functions as functions import parsimony.functions.nesterov.gl as gl import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl import parsimony.utils.start_vectors as start_vectors np.random.seed(42) # Note that p must be even! n, p = 25, 30 groups = [range(0, 2 * p / 3), range(p / 3, p)] weights = [1.5, 0.5] A = gl.A_from_groups(p, groups=groups, weights=weights) l = 0.618 k = 1.0 - l g = 2.718 start_vector = start_vectors.RandomStartVector(normalise=True) beta = start_vector.get_vector(p) alpha = 1.0 Sigma = alpha * np.eye(p, p) \ + (1.0 - alpha) * np.random.randn(p, p) mean = np.zeros(p) M = np.random.multivariate_normal(mean, Sigma, n) e = np.random.randn(n, 1) snr = 100.0 X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr) eps = 1e-8 max_iter = 10000 beta_start = start_vector.get_vector(p) mus = [5e-0, 5e-2, 5e-4, 5e-6, 5e-8] fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus)) beta_parsimony = beta_start for mu in mus: # function = functions.LinearRegressionL1L2GL(X, y, l, k, g, # A=A, mu=mu, # penalty_start=0) function = CombinedFunction() function.add_function(functions.losses.LinearRegression(X, y, mean=False)) function.add_penalty(functions.penalties.L2Squared(l=k)) function.add_penalty(gl.GroupLassoOverlap(l=g, A=A, mu=mu, penalty_start=0)) function.add_prox(functions.penalties.L1(l=l)) beta_parsimony = fista.run(function, beta_parsimony) berr = np.linalg.norm(beta_parsimony - beta_star) # print berr assert berr < 5e-3 f_parsimony = function.f(beta_parsimony) f_star = function.f(beta_star) # print abs(f_parsimony - f_star) assert abs(f_parsimony - f_star) < 5e-6
def test_DynamicCONESTA_gl(self): import numpy as np np.random.seed(42) import parsimony.estimators as estimators import parsimony.functions.nesterov.gl as gl import parsimony.utils.start_vectors as start_vectors import parsimony.algorithms.primaldual as primaldual import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl start_vector = start_vectors.RandomStartVector(normalise=True, limits=(-1, 1)) # Note that p should be divisible by 3! n, p = 75, 90 penalty_start = 0 groups = [range(penalty_start, 2 * p / 3), range(p / 3, p)] weights = [1.5, 0.5] l = 0.618 k = 1.0 - l g = 1.618 snr = 100.0 A = gl.A_from_groups(p, groups=groups, weights=weights, penalty_start=penalty_start) alpha = 0.9 Sigma = alpha * np.eye(p - penalty_start, p - penalty_start) \ + (1.0 - alpha) * np.random.randn(p - penalty_start, p - penalty_start) mean = np.zeros(p - penalty_start) M = np.random.multivariate_normal(mean, Sigma, n) if penalty_start > 0: M = np.hstack((np.ones((n, 1)), M)) e = np.random.randn(n, 1) while np.min(np.abs(np.dot(M.T, e))) < 1.0 / np.sqrt(n) \ or np.max(np.abs(np.dot(M.T, e))) > n: e = np.random.randn(n, 1) beta = start_vector.get_vector(p) beta = np.sort(beta, axis=0) beta[np.abs(beta) < 0.05] = 0.0 if penalty_start > 0: beta[0, 0] = 2.7182818 X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr, intercept=penalty_start > 0) eps = 1e-8 max_iter = 4200 mu = None dynamic = estimators.LinearRegressionL1L2GL(l, k, g, A, mu=mu, algorithm=primaldual.DynamicCONESTA(), algorithm_params=dict(eps=eps, max_iter=max_iter), penalty_start=penalty_start, mean=False) dynamic.fit(X, y) err = dynamic.score(X, y) # print "err :", err beta_dynamic = dynamic.beta dynamic.beta = beta_star err_star = dynamic.score(X, y) # print "err*:", err_star serr = abs(err - err_star) # print "score diff:", serr assert_less(serr, 5e-3, msg="The algorithm did not find a minimiser.") berr = np.linalg.norm(beta_dynamic - beta_star) # print "beta diff:", berr assert_less(berr, 5e-3, msg="The algorithm did not find a minimiser.")
def test_combo_overlapping_nonsmooth(self): import numpy as np from parsimony.functions import CombinedFunction import parsimony.algorithms.proximal as proximal import parsimony.functions as functions import parsimony.functions.nesterov.gl as gl import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl import parsimony.utils.weights as weights np.random.seed(42) # Note that p must be even! n, p = 25, 30 groups = [list(range(0, 2 * int(p / 3))), list(range(int(p / 3), p))] group_weights = [1.5, 0.5] A = gl.linear_operator_from_groups(p, groups=groups, weights=group_weights) l = 0.618 k = 1.0 - l g = 2.718 start_vector = weights.RandomUniformWeights(normalise=True) beta = start_vector.get_weights(p) alpha = 1.0 Sigma = alpha * np.eye(p, p) \ + (1.0 - alpha) * np.random.randn(p, p) mean = np.zeros(p) M = np.random.multivariate_normal(mean, Sigma, n) e = np.random.randn(n, 1) snr = 100.0 X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr) eps = 1e-8 max_iter = 10000 beta_start = start_vector.get_weights(p) mus = [5e-0, 5e-2, 5e-4, 5e-6, 5e-8] fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus)) beta_parsimony = beta_start for mu in mus: # function = functions.LinearRegressionL1L2GL(X, y, l, k, g, # A=A, mu=mu, # penalty_start=0) function = CombinedFunction() function.add_loss( functions.losses.LinearRegression(X, y, mean=False)) function.add_penalty(functions.penalties.L2Squared(l=k)) function.add_penalty( gl.GroupLassoOverlap(l=g, A=A, mu=mu, penalty_start=0)) function.add_prox(functions.penalties.L1(l=l)) beta_parsimony = fista.run(function, beta_parsimony) berr = np.linalg.norm(beta_parsimony - beta_star) # print berr assert berr < 5e-3 f_parsimony = function.f(beta_parsimony) f_star = function.f(beta_star) # print abs(f_parsimony - f_star) assert abs(f_parsimony - f_star) < 5e-6
def test_nonoverlapping_nonsmooth(self): # Spams: http://spams-devel.gforge.inria.fr/doc-python/doc_spams.pdf import numpy as np from parsimony.functions import CombinedFunction import parsimony.algorithms.proximal as proximal import parsimony.functions as functions import parsimony.functions.nesterov.gl as gl import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl import parsimony.utils.start_vectors as start_vectors np.random.seed(42) # Note that p must be even! n, p = 25, 20 groups = [list(range(0, int(p / 2))), list(range(int(p / 2), p))] # weights = [1.5, 0.5] A = gl.linear_operator_from_groups(p, groups=groups) # , weights=weights) l = 0.0 k = 0.0 g = 1.0 start_vector = start_vectors.RandomStartVector(normalise=True) beta = start_vector.get_vector(p) alpha = 1.0 Sigma = alpha * np.eye(p, p) \ + (1.0 - alpha) * np.random.randn(p, p) mean = np.zeros(p) M = np.random.multivariate_normal(mean, Sigma, n) e = np.random.randn(n, 1) snr = 100.0 X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr) eps = 1e-8 max_iter = 8500 beta_start = start_vector.get_vector(p) mus = [5e-2, 5e-4, 5e-6, 5e-8] fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus)) beta_parsimony = beta_start for mu in mus: # function = functions.LinearRegressionL1L2GL(X, y, l, k, g, # A=A, mu=mu, # penalty_start=0) function = CombinedFunction() function.add_function( functions.losses.LinearRegression(X, y, mean=False)) function.add_penalty( gl.GroupLassoOverlap(l=g, A=A, mu=mu, penalty_start=0)) beta_parsimony = fista.run(function, beta_parsimony) try: import spams params = { "loss": "square", "regul": "group-lasso-l2", "groups": np.array([1] * (int(p / 2)) + [2] * (int(p / 2)), dtype=np.int32), "lambda1": g, "max_it": max_iter, "tol": eps, "ista": False, "numThreads": -1, } beta_spams, optim_info = \ spams.fistaFlat(Y=np.asfortranarray(y), X=np.asfortranarray(X), W0=np.asfortranarray(beta_start), return_optim_info=True, **params) except ImportError: beta_spams = np.asarray( [[14.01111427], [35.56508563], [27.38245962], [22.39716553], [5.835744940], [5.841502910], [2.172209350], [32.40227785], [22.48364756], [26.48822401], [0.770391500], [36.28288883], [31.14118214], [7.938279340], [6.800713150], [6.862914540], [11.38161678], [19.63087584], [16.15855845], [10.89356615]]) berr = np.linalg.norm(beta_parsimony - beta_spams) # print berr assert berr < 5e-2 f_parsimony = function.f(beta_parsimony) f_spams = function.f(beta_spams) ferr = abs(f_parsimony - f_spams) # print ferr assert ferr < 5e-6
def test_overlapping_nonsmooth(self): import numpy as np from parsimony.functions import CombinedFunction import parsimony.algorithms.proximal as proximal import parsimony.functions as functions import parsimony.functions.nesterov.gl as gl import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl import parsimony.utils.weights as weights np.random.seed(42) # Note that p should be divisible by 3! n, p = 25, 30 groups = [list(range(0, 2 * int(p / 3))), list(range(int(p / 3), p))] group_weights = [1.5, 0.5] A = gl.linear_operator_from_groups(p, groups=groups, weights=group_weights) l = 0.0 k = 0.0 g = 1.1 start_vector = weights.RandomUniformWeights(normalise=True) beta = start_vector.get_weights(p) alpha = 1.0 Sigma = alpha * np.eye(p, p) \ + (1.0 - alpha) * np.random.randn(p, p) mean = np.zeros(p) M = np.random.multivariate_normal(mean, Sigma, n) e = np.random.randn(n, 1) snr = 100.0 X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr) eps = 1e-8 max_iter = 8000 beta_start = start_vector.get_weights(p) mus = [5e-2, 5e-4, 5e-6, 5e-8] fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus)) beta_parsimony = beta_start for mu in mus: # function = functions.LinearRegressionL1L2GL(X, y, l, k, g, # A=A, mu=mu, # penalty_start=0) function = CombinedFunction() function.add_loss(functions.losses.LinearRegression(X, y, mean=False)) function.add_penalty(gl.GroupLassoOverlap(l=g, A=A, mu=mu, penalty_start=0)) beta_parsimony = fista.run(function, beta_parsimony) berr = np.linalg.norm(beta_parsimony - beta_star) # print berr assert berr < 0.05 f_parsimony = function.f(beta_parsimony) f_star = function.f(beta_star) # print(abs(f_parsimony - f_star)) assert abs(f_parsimony - f_star) < 5e-4
def test_overlapping_nonsmooth(self): import numpy as np from parsimony.functions import CombinedFunction import parsimony.algorithms.proximal as proximal import parsimony.functions as functions import parsimony.functions.nesterov.gl as gl import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl import parsimony.utils.start_vectors as start_vectors np.random.seed(42) # Note that p should be divisible by 3! n, p = 25, 30 groups = [range(0, 2 * p / 3), range(p / 3, p)] weights = [1.5, 0.5] A = gl.linear_operator_from_groups(p, groups=groups, weights=weights) l = 0.0 k = 0.0 g = 1.1 start_vector = start_vectors.RandomStartVector(normalise=True) beta = start_vector.get_vector(p) alpha = 1.0 Sigma = alpha * np.eye(p, p) \ + (1.0 - alpha) * np.random.randn(p, p) mean = np.zeros(p) M = np.random.multivariate_normal(mean, Sigma, n) e = np.random.randn(n, 1) snr = 100.0 X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr) eps = 1e-8 max_iter = 8000 beta_start = start_vector.get_vector(p) mus = [5e-2, 5e-4, 5e-6, 5e-8] fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus)) beta_parsimony = beta_start for mu in mus: # function = functions.LinearRegressionL1L2GL(X, y, l, k, g, # A=A, mu=mu, # penalty_start=0) function = CombinedFunction() function.add_function( functions.losses.LinearRegression(X, y, mean=False)) function.add_penalty( gl.GroupLassoOverlap(l=g, A=A, mu=mu, penalty_start=0)) beta_parsimony = fista.run(function, beta_parsimony) berr = np.linalg.norm(beta_parsimony - beta_star) # print berr assert berr < 0.05 f_parsimony = function.f(beta_parsimony) f_star = function.f(beta_star) # print abs(f_parsimony - f_star) assert abs(f_parsimony - f_star) < 5e-5