def __init__(self, l1, l2, ltv, Atv, n_components, mu=None, criterion="frobenius", eps=consts.TOLERANCE, max_iter=10000, inner_eps=consts.TOLERANCE, inner_max_iter=10000, tau=0.2, output=False, start_vector=start_vectors.RandomStartVector(), raise_if_l1_too_large=True, callback=None): self.l1 = float(l1) self.l2 = float(l2) self.ltv = float(ltv) if (self.l2 <= consts.TOLERANCE): msg_fmt = "The ridge parameter must be > to consts.TOLERANCE ({0})" msg = msg_fmt.format(consts.TOLERANCE) raise ValueError(msg) if (self.ltv <= consts.TOLERANCE): msg_fmt = "The TV parameter must be > to consts.TOLERANCE ({0})" msg = msg_fmt.format(consts.TOLERANCE) raise ValueError(msg) self.Atv = Atv self.n_components = int(n_components) self.start_vector = start_vector try: self.mu = float(mu) except (ValueError, TypeError): self.mu = None # Stopping criterion if criterion not in PCA_L1_L2_TV.CRITERIA: raise ValueError self.criterion = criterion self.eps = float(eps) self.max_iter = max_iter # Inner optimization criteria self.inner_eps = inner_eps self.inner_max_iter = inner_max_iter self.tau = tau self.raise_if_l1_too_large = raise_if_l1_too_large self.output = output self.callback = callback self.lmaxA = self.lambda_max() # Call parent init # We don't initialize the algorithm here super(PCA_L1_L2_TV, self).__init__(algorithm=None)
def run(self, X, Y, start_vector=None): """Find the right-singular vector of the product of two matrices. Parameters ---------- X : Numpy array with shape (n, p). The first matrix of the product. Y : Numpy array with shape (p, m). The second matrix of the product. start_vector : BaseStartVector. A start vector generator. Default is to use a random start vector. """ if self.info_requested(utils.Info.ok): self.info_set(utils.Info.ok, False) if self.info_requested(utils.Info.time): _t = utils.time() if self.info_requested(utils.Info.converged): self.info_set(utils.Info.converged, False) M, N = X.shape if start_vector is None: start_vector = start_vectors.RandomStartVector(normalise=True) v = start_vector.get_vector(Y.shape[1]) for it in xrange(1, self.max_iter + 1): v_ = v v = np.dot(X, np.dot(Y, v_)) v = np.dot(Y.T, np.dot(X.T, v)) v *= 1.0 / maths.norm(v) if maths.norm(v_ - v) / maths.norm(v) < self.eps \ and it >= self.min_iter: if self.info_requested(utils.Info.converged): self.info_set(utils.Info.converged, True) break if self.info_requested(utils.Info.time): self.info_set(utils.Info.time, utils.time() - _t) if self.info_requested(utils.Info.func_val): _f = maths.norm(np.dot(X, np.dot(Y, v))) # Largest singular value. self.info_set(utils.Info.func_val, _f) if self.info_requested(utils.Info.ok): self.info_set(utils.Info.ok, True) return utils.direct_vector(v)
def __init__(self, l, mean=True, penalty_start=0, start_vector=start_vectors.RandomStartVector( limits=(-1.0, 1.0)), eps=consts.TOLERANCE, info=[], max_iter=10000, min_iter=1): super(LassoCoordinateDescent, self).__init__(info=info, max_iter=max_iter, min_iter=min_iter) self.l = max(0.0, float(l)) self.mean = bool(mean) self.penalty_start = max(0, int(penalty_start)) self.start_vector = start_vector self.eps = max(consts.TOLERANCE, float(eps))
def test_nonsmooth(self): import numpy as np import parsimony.utils.consts as consts from parsimony.functions import CombinedFunction import parsimony.algorithms.proximal as proximal import parsimony.functions.losses as losses import parsimony.functions.penalties as penalties import parsimony.functions.nesterov as nesterov import parsimony.utils.start_vectors as start_vectors import parsimony.datasets.simulate.l1_l2_tv as l1_l2_tv start_vector = start_vectors.RandomStartVector(normalise=True) np.random.seed(42) n, p = 75, 100 alpha = 0.9 Sigma = alpha * np.eye(p, p) \ + (1.0 - alpha) * np.random.randn(p, p) mean = np.zeros(p) M = np.random.multivariate_normal(mean, Sigma, n) e = np.random.randn(n, 1) beta_start = start_vector.get_vector(p) beta_start[np.abs(beta_start) < 0.1] = 0.0 l = 0.618 k = 0.0 g = 0.0 A = np.eye(p) A = [A, A, A] snr = 100.0 X, y, beta_star = l1_l2_tv.load(l, k, g, beta_start, M, e, A, snr=snr) beta = beta_start for mu in [5e-2, 5e-3, 5e-4, 5e-5]: function = CombinedFunction() function.add_function(losses.LinearRegression(X, y, mean=False)) A = nesterov.l1.linear_operator_from_variables(p, penalty_start=0) function.add_penalty(nesterov.l1.L1(l, A=A, mu=mu, penalty_start=0)) fista = proximal.FISTA(eps=consts.TOLERANCE, max_iter=910) beta = fista.run(function, beta) berr = np.linalg.norm(beta - beta_star) # print "berr:", berr assert berr < 5e-2 # Test proximal operator beta = beta_start function = CombinedFunction() function.add_function(losses.LinearRegression(X, y, mean=False)) A = nesterov.l1.linear_operator_from_variables(p, penalty_start=0) # function.add_penalty(nesterov.l1.L1(l, A=A, mu=mu_min, # penalty_start=penalty_start)) function.add_prox(nesterov.l1.L1(l, A=A, mu=5e-5, penalty_start=0)) fista = proximal.FISTA(eps=consts.TOLERANCE, max_iter=800) beta = fista.run(function, beta) berr = np.linalg.norm(beta - beta_star) # print "berr:", berr assert berr < 5e-0
def test_smoothed_l1tv(self): import numpy as np from parsimony.functions import CombinedFunction import parsimony.algorithms.proximal as proximal import parsimony.functions as functions import parsimony.functions.penalties as penalties import parsimony.functions.nesterov.tv as tv import parsimony.functions.nesterov.l1tv as l1tv import parsimony.datasets.simulate.l1_l2_tvmu as l1_l2_tvmu import parsimony.utils.start_vectors as start_vectors import parsimony.datasets.simulate as simulate np.random.seed(42) px = 10 py = 1 pz = 1 shape = (pz, py, px) n, p = 5, np.prod(shape) l = 0.618 k = 0.01 g = 1.1 start_vector = start_vectors.RandomStartVector(normalise=True) beta = start_vector.get_vector(p) alpha = 1.0 Sigma = alpha * np.eye(p, p) \ + (1.0 - alpha) * np.random.randn(p, p) mean = np.zeros(p) M = np.random.multivariate_normal(mean, Sigma, n) e = np.random.randn(n, 1) snr = 100.0 mu = 5e-3 A = tv.linear_operator_from_shape(shape) # X, y, beta_star = l1_l2_tvmu.load(l=l, k=k, g=g, beta=beta, M=M, e=e, # A=A, mu=mu, snr=snr) funs = [ simulate.grad.L1(l), simulate.grad.L2Squared(k), simulate.grad.TotalVariation(g, A) ] lr = simulate.LinearRegressionData(funs, M, e, snr=snr, intercept=False) X, y, beta_star = lr.load(beta) eps = 1e-8 max_iter = 810 alg = proximal.FISTA(eps=eps, max_iter=max_iter) function = CombinedFunction() function.add_function( functions.losses.LinearRegression(X, y, mean=False)) function.add_penalty(penalties.L2Squared(l=k)) A = l1tv.linear_operator_from_shape(shape, p) function.add_prox(l1tv.L1TV(l, g, A=A, mu=mu, penalty_start=0)) # A = tv.linear_operator_from_shape(shape) # function.add_penalty(tv.TotalVariation(l=g, A=A, mu=mu, # penalty_start=0)) # function.add_prox(penalties.L1(l=l)) beta_start = start_vector.get_vector(p) beta = alg.run(function, beta_start) berr = np.linalg.norm(beta - beta_star) # print "berr:", berr assert berr < 5e-1 f_parsimony = function.f(beta) f_star = function.f(beta_star) ferr = abs(f_parsimony - f_star) # print "ferr:", ferr assert ferr < 5e-3
def test_combo_overlapping_nonsmooth(self): import numpy as np from parsimony.functions import CombinedFunction import parsimony.algorithms.proximal as proximal import parsimony.functions as functions import parsimony.functions.nesterov.gl as gl import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl import parsimony.utils.start_vectors as start_vectors np.random.seed(42) # Note that p must be even! n, p = 25, 30 groups = [list(range(0, 2 * int(p / 3))), list(range(int(p / 3), p))] weights = [1.5, 0.5] A = gl.linear_operator_from_groups(p, groups=groups, weights=weights) l = 0.618 k = 1.0 - l g = 2.718 start_vector = start_vectors.RandomStartVector(normalise=True) beta = start_vector.get_vector(p) alpha = 1.0 Sigma = alpha * np.eye(p, p) \ + (1.0 - alpha) * np.random.randn(p, p) mean = np.zeros(p) M = np.random.multivariate_normal(mean, Sigma, n) e = np.random.randn(n, 1) snr = 100.0 X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr) eps = 1e-8 max_iter = 10000 beta_start = start_vector.get_vector(p) mus = [5e-0, 5e-2, 5e-4, 5e-6, 5e-8] fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus)) beta_parsimony = beta_start for mu in mus: # function = functions.LinearRegressionL1L2GL(X, y, l, k, g, # A=A, mu=mu, # penalty_start=0) function = CombinedFunction() function.add_function( functions.losses.LinearRegression(X, y, mean=False)) function.add_penalty(functions.penalties.L2Squared(l=k)) function.add_penalty( gl.GroupLassoOverlap(l=g, A=A, mu=mu, penalty_start=0)) function.add_prox(functions.penalties.L1(l=l)) beta_parsimony = fista.run(function, beta_parsimony) berr = np.linalg.norm(beta_parsimony - beta_star) # print berr assert berr < 5e-3 f_parsimony = function.f(beta_parsimony) f_star = function.f(beta_star) # print abs(f_parsimony - f_star) assert abs(f_parsimony - f_star) < 5e-6
def test_nonoverlapping_nonsmooth(self): # Spams: http://spams-devel.gforge.inria.fr/doc-python/doc_spams.pdf import numpy as np from parsimony.functions import CombinedFunction import parsimony.algorithms.proximal as proximal import parsimony.functions as functions import parsimony.functions.nesterov.gl as gl import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl import parsimony.utils.start_vectors as start_vectors np.random.seed(42) # Note that p must be even! n, p = 25, 20 groups = [list(range(0, int(p / 2))), list(range(int(p / 2), p))] # weights = [1.5, 0.5] A = gl.linear_operator_from_groups(p, groups=groups) # , weights=weights) l = 0.0 k = 0.0 g = 1.0 start_vector = start_vectors.RandomStartVector(normalise=True) beta = start_vector.get_vector(p) alpha = 1.0 Sigma = alpha * np.eye(p, p) \ + (1.0 - alpha) * np.random.randn(p, p) mean = np.zeros(p) M = np.random.multivariate_normal(mean, Sigma, n) e = np.random.randn(n, 1) snr = 100.0 X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr) eps = 1e-8 max_iter = 8500 beta_start = start_vector.get_vector(p) mus = [5e-2, 5e-4, 5e-6, 5e-8] fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus)) beta_parsimony = beta_start for mu in mus: # function = functions.LinearRegressionL1L2GL(X, y, l, k, g, # A=A, mu=mu, # penalty_start=0) function = CombinedFunction() function.add_function( functions.losses.LinearRegression(X, y, mean=False)) function.add_penalty( gl.GroupLassoOverlap(l=g, A=A, mu=mu, penalty_start=0)) beta_parsimony = fista.run(function, beta_parsimony) try: import spams params = { "loss": "square", "regul": "group-lasso-l2", "groups": np.array([1] * (int(p / 2)) + [2] * (int(p / 2)), dtype=np.int32), "lambda1": g, "max_it": max_iter, "tol": eps, "ista": False, "numThreads": -1, } beta_spams, optim_info = \ spams.fistaFlat(Y=np.asfortranarray(y), X=np.asfortranarray(X), W0=np.asfortranarray(beta_start), return_optim_info=True, **params) except ImportError: beta_spams = np.asarray( [[14.01111427], [35.56508563], [27.38245962], [22.39716553], [5.835744940], [5.841502910], [2.172209350], [32.40227785], [22.48364756], [26.48822401], [0.770391500], [36.28288883], [31.14118214], [7.938279340], [6.800713150], [6.862914540], [11.38161678], [19.63087584], [16.15855845], [10.89356615]]) berr = np.linalg.norm(beta_parsimony - beta_spams) # print berr assert berr < 5e-2 f_parsimony = function.f(beta_parsimony) f_spams = function.f(beta_spams) ferr = abs(f_parsimony - f_spams) # print ferr assert ferr < 5e-6
def test_nonoverlapping_smooth(self): # Spams: http://spams-devel.gforge.inria.fr/doc-python/doc_spams.pdf import numpy as np from parsimony.functions import CombinedFunction import parsimony.algorithms.proximal as proximal import parsimony.functions as functions import parsimony.functions.nesterov.gl as gl import parsimony.datasets.simulate.l1_l2_glmu as l1_l2_glmu import parsimony.utils.start_vectors as start_vectors np.random.seed(42) # Note that p must be even! n, p = 25, 20 groups = [list(range(0, int(p / 2))), list(range(int(p / 2), p))] # weights = [1.5, 0.5] A = gl.linear_operator_from_groups(p, groups=groups) # , weights=weights) l = 0.0 k = 0.0 g = 0.9 start_vector = start_vectors.RandomStartVector(normalise=True) beta = start_vector.get_vector(p) alpha = 1.0 Sigma = alpha * np.eye(p, p) \ + (1.0 - alpha) * np.random.randn(p, p) mean = np.zeros(p) M = np.random.multivariate_normal(mean, Sigma, n) e = np.random.randn(n, 1) snr = 100.0 mu_min = 5e-8 X, y, beta_star = l1_l2_glmu.load(l, k, g, beta, M, e, A, mu=mu_min, snr=snr) eps = 1e-8 max_iter = 18000 beta_start = start_vector.get_vector(p) mus = [5e-0, 5e-2, 5e-4, 5e-6, 5e-8] fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus)) beta_parsimony = beta_start for mu in mus: # function = functions.LinearRegressionL1L2GL(X, y, l, k, g, # A=A, mu=mu, # penalty_start=0) function = CombinedFunction() function.add_function( functions.losses.LinearRegression(X, y, mean=False)) function.add_penalty( gl.GroupLassoOverlap(l=g, A=A, mu=mu, penalty_start=0)) beta_parsimony = fista.run(function, beta_parsimony) try: import spams params = { "loss": "square", "regul": "group-lasso-l2", "groups": np.array([1] * (int(p / 2)) + [2] * (int(p / 2)), dtype=np.int32), "lambda1": g, "max_it": max_iter, "tol": eps, "ista": False, "numThreads": -1, } beta_spams, optim_info = \ spams.fistaFlat(Y=np.asfortranarray(y), X=np.asfortranarray(X), W0=np.asfortranarray(beta_start), return_optim_info=True, **params) # print beta_spams except ImportError: beta_spams = np.asarray( [[15.56784201], [39.51679274], [30.42583205], [24.8816362], [6.48671072], [6.48350546], [2.41477318], [36.00285723], [24.98522184], [29.43128643], [0.85520539], [40.31463542], [34.60084146], [8.82322513], [7.55741642], [7.62364398], [12.64594707], [21.81113869], [17.95400007], [12.10507338]]) berr = np.linalg.norm(beta_parsimony - beta_spams) # print berr assert berr < 5e-3 f_parsimony = function.f(beta_parsimony) f_spams = function.f(beta_spams) ferr = abs(f_parsimony - f_spams) # print ferr assert ferr < 5e-6
def run(self, X, start_vector=None): """Find the right-singular vector of the given matrix. Parameters ---------- X : Numpy array. The matrix to decompose. start_vector : BaseStartVector. A start vector generator. Default is to use a random start vector. """ if self.info_requested(utils.Info.ok): self.info_set(utils.Info.ok, False) if self.info_requested(utils.Info.time): _t = utils.time() if start_vector is None: start_vector = start_vectors.RandomStartVector(normalise=True) v0 = start_vector.get_vector(np.min(X.shape)) arpack_failed = False try: try: [_, _, v] = sparse_linalg.svds(X, k=1, v0=v0, tol=self.eps, maxiter=self.max_iter, return_singular_vectors=True) except TypeError: # For scipy 0.9.0. [_, _, v] = sparse_linalg.svds(X, k=1, tol=self.eps) v = v.T if self.info_requested(utils.Info.converged): self.info_set(utils.Info.converged, True) except ArpackNoConvergence: arpack_failed = True if arpack_failed: # Use the power method if this happens. M, N = X.shape if M < 80 and N < 80: # Very arbitrary threshold from one computer _, _, V = scipy.linalg.svd(X, full_matrices=True) v = V[[0], :].T elif M < N: K = np.dot(X, X.T) t = v0 for it in xrange(self.max_iter): t_ = t t = np.dot(K, t_) t *= 1.0 / maths.norm(t) if maths.norm(t_ - t) / maths.norm(t) < self.eps: break v = np.dot(X.T, t) v *= 1.0 / maths.norm(v) else: K = np.dot(X.T, X) v = v0 for it in xrange(self.max_iter): v_ = v v = np.dot(K, v_) v *= 1.0 / maths.norm(v) if maths.norm(v_ - v) / maths.norm(v) < self.eps: break if self.info_requested(utils.Info.time): self.info_set(utils.Info.time, utils.time() - _t) if self.info_requested(utils.Info.func_val): _f = maths.norm(np.dot(X, v)) # Largest singular value. self.info_set(utils.Info.func_val, _f) if self.info_requested(utils.Info.ok): self.info_set(utils.Info.ok, True) return utils.direct_vector(v)
range(int(2.0 * p / 3.0), p) ]) snr = 20.0 mu = 1e-6 #X, y, beta_star = simulated.l1_l2_tvmu.load(l, k, g, beta_star, M, e, Atv, mu, snr) #X, y, beta_star = simulated.l1_l2_tv.load(l, k, g, beta_star, M, e, Atv, snr) #X, y, beta_star = simulated.l1_l2_glmu.load(l, k, g, beta_star, M, e, Agl, mu, snr) X, y, beta_star = simulated.l1_l2_gl.load(l, k, g, beta_star, M, e, Agl, snr) f = [] fmu = [] b = [] start_vector = start_vectors.RandomStartVector() errs = [-0.1, -0.08, -0.06, -0.04, -0.02, 0.0, 0.02, 0.04, 0.06, 0.08, 0.1] for er in errs: # function = functions.RR_L1_TV(X, y, k, l, g + er, A=Atv, mu=mu) function = functions.RR_L1_GL(X, y, k, l, g + er, A=Agl, mu=mu) beta = start_vector.get_vector(X.shape[1]) start_vector = start_vectors.IdentityStartVector(beta) # conts = 10 # algorithm = algorithms.StaticCONESTA(mu_start=mu * 2.0 ** (conts + 6), # output=True, # continuations=conts, max_iter=100000) algorithm = algorithms.FISTA(output=True, max_iter=200000)
def test_smoothed(self): import numpy as np from parsimony.functions import CombinedFunction import parsimony.algorithms.proximal as proximal import parsimony.functions.losses as losses import parsimony.functions.nesterov as nesterov import parsimony.utils.start_vectors as start_vectors import parsimony.datasets.simulate.l1_l2_tv as l1_l2_tv start_vector = start_vectors.RandomStartVector(normalise=True) np.random.seed(42) n, p = 75, 100 penalty_start = 0 alpha = 0.9 V = np.random.randn(p, p) Sigma = alpha * np.eye(p, p) \ + (1.0 - alpha) * np.dot(V.T, V) mean = np.zeros(p) M = np.random.multivariate_normal(mean, Sigma, n) e = np.random.randn(n, 1) beta = start_vector.get_vector(p) beta[np.abs(beta) < 0.1] = 0.0 l = 0.618 k = 0.0 g = 0.0 mu_min = 0.001 # consts.TOLERANCE A = np.eye(p) A = [A, A, A] snr = 100.0 X, y, beta_star = l1_l2_tv.load(l, k, g, beta, M, e, A, snr=snr) function = CombinedFunction() function.add_loss(losses.LinearRegression(X, y, mean=False)) A = nesterov.l1.linear_operator_from_variables( p, penalty_start=penalty_start) function.add_penalty( nesterov.l1.L1(l, A=A, mu=mu_min, penalty_start=penalty_start)) # function.add_prox(penalties.L1(l, penalty_start=penalty_start)) fista = proximal.FISTA(eps=mu_min, max_iter=23500) beta = fista.run(function, beta) berr = np.linalg.norm(beta - beta_star) # print "berr:", berr # assert berr < 5 assert_less(berr, 5.0, "The found regression vector is not correct.") # Test proximal operator function = CombinedFunction() function.add_loss(losses.LinearRegression(X, y, mean=False)) A = nesterov.l1.linear_operator_from_variables( p, penalty_start=penalty_start) function.add_prox( nesterov.l1.L1(l, A=A, mu=mu_min, penalty_start=penalty_start)) fista = proximal.FISTA(eps=mu_min, max_iter=20000) beta = fista.run(function, beta) berr = np.linalg.norm(beta - beta_star) # print "berr:", berr # assert berr < 0.1 assert_less(berr, 0.1, "The found regression vector is not correct.")
X= np.load('/neurospin/brainomics/2014_pca_struct/synthetic_data/data_100_100/data.npy') SHAPE = (100, 100, 1) # a, l1, l2, tv penalties global_pen = 0.01 tv_ratio = 0.5#1e-05 l1_ratio = 0.5 ltv = global_pen * tv_ratio ll1 = l1_ratio * global_pen * (1 - tv_ratio) ll2 = (1 - l1_ratio) * global_pen * (1 - tv_ratio) assert(np.allclose(ll1 + ll2 + ltv, global_pen)) Atv = nesterov_tv.A_from_shape(SHAPE) start_vector=start_vectors.RandomStartVector(seed=42) ############################################################################## snapshot = AlgorithmSnapshot('/neurospin/brainomics/2014_pca_struct/synthetic_data/data_100_100_bis/').save_nipals t0 = utils.time_cpu() mod = pca_tv.PCA_L1_L2_TV(n_components=3, l1=ll1, l2=ll2, ltv=ltv, Atv=Atv, criterion="frobenius", eps=1e-4, max_iter=100, inner_max_iter=int(1e4), output=True,start_vector=start_vector,callback=snapshot)
############################################################################### ## Models ############################################################################### MODELS = collections.OrderedDict() N_COMP = 3 im_shape = (100,100) Atv = parsimony.functions.nesterov.tv.A_from_shape(im_shape) # l2, l1, tv penalties global_pen=0.01 l1_ratio=0.5 tv_ratio=0.5 ll1, ll2, ltv = compute_coefs_from_ratios(global_pen,tv_ratio,l1_ratio) start_vector = start_vectors.RandomStartVector(seed=24) MODELS["SparsePCA"] = \ sklearn.decomposition.SparsePCA(n_components=N_COMP,alpha=1) MODELS["ElasticNetPCA"] = \ pca_tv.PCA_L1_L2_TV(n_components=N_COMP, l1=ll1, l2=ll2, ltv=1e-6, Atv=Atv, criterion="frobenius", eps=1e-6, max_iter=100, inner_max_iter=int(1e4), output=False,start_vector=start_vector)
# Load true weights if has_data: WEIGHTS_TRUTH = np.load(weights_filename(shape, n_samples)) # Ensure that train dataset is balanced tr = np.hstack([np.where(y.ravel() == 1)[0][:int(n_train / 2)], np.where(y.ravel() == 0)[0][:int(n_train / 2)]]) te = np.setdiff1d(np.arange(y.shape[0]), tr) X = X3d.reshape((n_samples, np.prod(beta3d.shape))) Xtr = X[tr, :] ytr = y[tr] Xte = X[te, :] yte = y[te] beta_start = start_vectors.RandomStartVector().get_vector(Xtr.shape[1]) # check that ytr is balanced #assert ytr.sum() / ytr.shape[0] == 0.5 #assert yte.sum() / yte.shape[0] == 0.53500000000000003 # Dataset with intercept Xtr_i = np.c_[np.ones((Xtr.shape[0], 1)), Xtr] Xte_i = np.c_[np.ones((Xte.shape[0], 1)), Xte] beta_start_i = start_vectors.RandomStartVector().get_vector(Xtr_i.shape[1]) # global penalty alpha = l1_max_logistic_loss(Xtr, ytr) from parsimony.algorithms.utils import Info info = [Info.converged,
def test_combo_smooth(self): from parsimony.functions import CombinedFunction import parsimony.algorithms.proximal as proximal import parsimony.functions as functions import parsimony.functions.nesterov.tv as tv import parsimony.datasets.simulate.l1_l2_tvmu as l1_l2_tvmu import parsimony.utils.start_vectors as start_vectors np.random.seed(42) px = 4 py = 4 pz = 4 shape = (pz, py, px) n, p = 50, np.prod(shape) l = 0.618 k = 1.0 - l g = 1.1 start_vector = start_vectors.RandomStartVector(normalise=True) beta = start_vector.get_vector(p) alpha = 1.0 Sigma = alpha * np.eye(p, p) \ + (1.0 - alpha) * np.random.randn(p, p) mean = np.zeros(p) M = np.random.multivariate_normal(mean, Sigma, n) e = np.random.randn(n, 1) snr = 100.0 A = tv.linear_operator_from_shape(shape) mu_min = 5e-8 X, y, beta_star = l1_l2_tvmu.load(l=l, k=k, g=g, beta=beta, M=M, e=e, A=A, mu=mu_min, snr=snr) eps = 1e-8 max_iter = 5300 beta_start = start_vector.get_vector(p) mus = [5e-2, 5e-4, 5e-6, 5e-8] fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus)) beta_parsimony = beta_start for mu in mus: # function = functions.LinearRegressionL1L2GL(X, y, l, k, g, # A=A, mu=mu, # penalty_start=0) function = CombinedFunction() function.add_function(functions.losses.LinearRegression(X, y, mean=False)) function.add_penalty(tv.TotalVariation(l=g, A=A, mu=mu, penalty_start=0)) function.add_penalty(functions.penalties.L2Squared(l=k)) function.add_prox(functions.penalties.L1(l=l)) beta_parsimony = fista.run(function, beta_parsimony) berr = np.linalg.norm(beta_parsimony - beta_star) # print "berr:", berr assert berr < 5e-3 f_parsimony = function.f(beta_parsimony) f_star = function.f(beta_star) ferr = abs(f_parsimony - f_star) # print "ferr:", ferr assert ferr < 5e-5
def test_smooth_1D_l2(self): from parsimony.functions import CombinedFunction import parsimony.functions as functions import parsimony.functions.nesterov.grouptv as grouptv import parsimony.datasets.simulate.l1_l2_grouptvmu as l1_l2_grouptvmu import parsimony.utils.start_vectors as start_vectors np.random.seed(1337) n, p = 10, 15 shape = (1, 1, p) l = 0.0 k = 0.1 # Must have some regularisation for all variables. g = 0.9 start_vector = start_vectors.RandomStartVector(normalise=True) beta = start_vector.get_vector(p) rects = [[(0, 5)], [(4, 10)], [(13, 15)]] # 0 [ 5 ] 0 # 1 [ 5 ] 0 # 2 [ 5 ] 0 # 3 [ 5 ] 0 # 4 [ 4 ] 0 / 1 beta[:5, :] = 5.0 # 5 [ 3 ] 1 beta[4, :] = 4.0 # 6 [ 3 ] 1 beta[5:10, :] = 3.0 # 7 [ 3 ] 1 beta[13:15, :] = 7.0 # 8 [ 3 ] 1 # 9 [ 3 ] 1 # 0 [ x ] - # 1 [ x ] - # 2 [ x ] - # 3 [ 7 ] 2 # 4 [ 7 ] 2 alpha = 1.0 Sigma = alpha * np.eye(p, p) \ + (1.0 - alpha) * np.random.randn(p, p) mean = np.zeros(p) M = np.random.multivariate_normal(mean, Sigma, n) e = np.random.randn(n, 1) snr = 100.0 A = grouptv.linear_operator_from_rects(rects, shape) mu_min = 5e-8 X, y, beta_star = l1_l2_grouptvmu.load(l=l, k=k, g=g, beta=beta, M=M, e=e, A=A, mu=mu_min, snr=snr) eps = 1e-5 max_iter = 12000 beta_start = start_vector.get_vector(p) mus = [5e-2, 5e-4, 5e-6, 5e-8] fista = FISTA(eps=eps, max_iter=max_iter / len(mus)) beta_parsimony = beta_start for mu in mus: function = CombinedFunction() function.add_function( functions.losses.LinearRegression(X, y, mean=False)) function.add_penalty( grouptv.GroupTotalVariation(l=g, A=A, mu=mu, penalty_start=0)) function.add_penalty( functions.penalties.L2Squared(l=k, penalty_start=0)) beta_parsimony = fista.run(function, beta_parsimony) berr = np.linalg.norm(beta_parsimony - beta_star) # print "berr:", berr assert berr < 5e-2 f_parsimony = function.f(beta_parsimony) f_star = function.f(beta_star) ferr = abs(f_parsimony - f_star) # print "ferr:", ferr assert ferr < 5e-5
def run(self, X, start_vector=None): """Find the right-singular vector of the given sparse matrix. Parameters ---------- X : Scipy sparse array. The sparse matrix to decompose. start_vector : BaseStartVector. A start vector generator. Default is to use a random start vector. """ if self.info_requested(utils.Info.ok): self.info_set(utils.Info.ok, False) if self.info_requested(utils.Info.time): _t = utils.time() if self.info_requested(utils.Info.converged): self.info_set(utils.Info.converged, False) if start_vector is None: start_vector = start_vectors.RandomStartVector(normalise=True) v0 = start_vector.get_vector(np.min(X.shape)) # determine when to use power method or scipy_sparse use_power = True if X.shape[1] >= 10**3 else False if not use_power: try: if not sp.sparse.issparse(X): X = sp.sparse.csr_matrix(X) try: [_, _, v] = sparse_linalg.svds(X, k=1, v0=v0, tol=self.eps, maxiter=self.max_iter, return_singular_vectors=True) except TypeError: # For scipy 0.9.0. [_, _, v] = sparse_linalg.svds(X, k=1, tol=self.eps) v = v.T if self.info_requested(utils.Info.converged): self.info_set(utils.Info.converged, True) except ArpackNoConvergence: use_power = True if use_power: # Use the power method if scipy failed or if determined. M, N = X.shape if M < N: K = X.dot(X.T) t = v0 for it in xrange(self.max_iter): t_ = t t = K.dot(t_) t *= 1.0 / maths.norm(t) crit = float(maths.norm(t_ - t)) / float(maths.norm(t)) if crit < consts.TOLERANCE: if self.info_requested(utils.Info.converged): self.info_set(utils.Info.converged, True) break v = X.T.dot(t) v *= 1.0 / maths.norm(v) else: K = X.T.dot(X) v = v0 for it in xrange(self.max_iter): v_ = v v = K.dot(v_) v *= 1.0 / maths.norm(v) crit = float(maths.norm(v_ - v)) / float(maths.norm(v)) if crit < consts.TOLERANCE: if self.info_requested(utils.Info.converged): self.info_set(utils.Info.converged, True) break if self.info_requested(utils.Info.time): self.info_set(utils.Info.time, utils.time() - _t) if self.info_requested(utils.Info.func_val): _f = maths.norm(X.dot(v)) # Largest singular value. self.info_set(utils.Info.func_val, _f) if self.info_requested(utils.Info.ok): self.info_set(utils.Info.ok, True) return utils.direct_vector(v)