def __init__(self,
l1,
l2,
ltv,
Atv,
n_components,
mu=None,
criterion="frobenius",
eps=consts.TOLERANCE,
max_iter=10000,
inner_eps=consts.TOLERANCE,
inner_max_iter=10000,
tau=0.2,
output=False,
start_vector=start_vectors.RandomStartVector(),
raise_if_l1_too_large=True,
callback=None):
self.l1 = float(l1)
self.l2 = float(l2)
self.ltv = float(ltv)
if (self.l2 <= consts.TOLERANCE):
msg_fmt = "The ridge parameter must be > to consts.TOLERANCE ({0})"
msg = msg_fmt.format(consts.TOLERANCE)
raise ValueError(msg)
if (self.ltv <= consts.TOLERANCE):
msg_fmt = "The TV parameter must be > to consts.TOLERANCE ({0})"
msg = msg_fmt.format(consts.TOLERANCE)
raise ValueError(msg)
self.Atv = Atv
self.n_components = int(n_components)
self.start_vector = start_vector
try:
self.mu = float(mu)
except (ValueError, TypeError):
self.mu = None
# Stopping criterion
if criterion not in PCA_L1_L2_TV.CRITERIA:
raise ValueError
self.criterion = criterion
self.eps = float(eps)
self.max_iter = max_iter
# Inner optimization criteria
self.inner_eps = inner_eps
self.inner_max_iter = inner_max_iter
self.tau = tau
self.raise_if_l1_too_large = raise_if_l1_too_large
self.output = output
self.callback = callback
self.lmaxA = self.lambda_max()
# Call parent init
# We don't initialize the algorithm here
super(PCA_L1_L2_TV, self).__init__(algorithm=None)
def run(self, X, Y, start_vector=None):
"""Find the right-singular vector of the product of two matrices.
Parameters
----------
X : Numpy array with shape (n, p). The first matrix of the product.
Y : Numpy array with shape (p, m). The second matrix of the product.
start_vector : BaseStartVector. A start vector generator. Default is to
use a random start vector.
"""
if self.info_requested(utils.Info.ok):
self.info_set(utils.Info.ok, False)
if self.info_requested(utils.Info.time):
_t = utils.time()
if self.info_requested(utils.Info.converged):
self.info_set(utils.Info.converged, False)
M, N = X.shape
if start_vector is None:
start_vector = start_vectors.RandomStartVector(normalise=True)
v = start_vector.get_vector(Y.shape[1])
for it in xrange(1, self.max_iter + 1):
v_ = v
v = np.dot(X, np.dot(Y, v_))
v = np.dot(Y.T, np.dot(X.T, v))
v *= 1.0 / maths.norm(v)
if maths.norm(v_ - v) / maths.norm(v) < self.eps \
and it >= self.min_iter:
if self.info_requested(utils.Info.converged):
self.info_set(utils.Info.converged, True)
break
if self.info_requested(utils.Info.time):
self.info_set(utils.Info.time, utils.time() - _t)
if self.info_requested(utils.Info.func_val):
_f = maths.norm(np.dot(X, np.dot(Y, v))) # Largest singular value.
self.info_set(utils.Info.func_val, _f)
if self.info_requested(utils.Info.ok):
self.info_set(utils.Info.ok, True)
return utils.direct_vector(v)
def __init__(self, l, mean=True, penalty_start=0,
start_vector=start_vectors.RandomStartVector(
limits=(-1.0, 1.0)),
eps=consts.TOLERANCE,
info=[], max_iter=10000, min_iter=1):
super(LassoCoordinateDescent, self).__init__(info=info,
max_iter=max_iter,
min_iter=min_iter)
self.l = max(0.0, float(l))
self.mean = bool(mean)
self.penalty_start = max(0, int(penalty_start))
self.start_vector = start_vector
self.eps = max(consts.TOLERANCE, float(eps))
def test_nonsmooth(self):
import numpy as np
import parsimony.utils.consts as consts
from parsimony.functions import CombinedFunction
import parsimony.algorithms.proximal as proximal
import parsimony.functions.losses as losses
import parsimony.functions.penalties as penalties
import parsimony.functions.nesterov as nesterov
import parsimony.utils.start_vectors as start_vectors
import parsimony.datasets.simulate.l1_l2_tv as l1_l2_tv
start_vector = start_vectors.RandomStartVector(normalise=True)
np.random.seed(42)
n, p = 75, 100
alpha = 0.9
Sigma = alpha * np.eye(p, p) \
+ (1.0 - alpha) * np.random.randn(p, p)
mean = np.zeros(p)
M = np.random.multivariate_normal(mean, Sigma, n)
e = np.random.randn(n, 1)
beta_start = start_vector.get_vector(p)
beta_start[np.abs(beta_start) < 0.1] = 0.0
l = 0.618
k = 0.0
g = 0.0
A = np.eye(p)
A = [A, A, A]
snr = 100.0
X, y, beta_star = l1_l2_tv.load(l, k, g, beta_start, M, e, A, snr=snr)
beta = beta_start
for mu in [5e-2, 5e-3, 5e-4, 5e-5]:
function = CombinedFunction()
function.add_function(losses.LinearRegression(X, y, mean=False))
A = nesterov.l1.linear_operator_from_variables(p, penalty_start=0)
function.add_penalty(nesterov.l1.L1(l, A=A, mu=mu,
penalty_start=0))
fista = proximal.FISTA(eps=consts.TOLERANCE, max_iter=910)
beta = fista.run(function, beta)
berr = np.linalg.norm(beta - beta_star)
# print "berr:", berr
assert berr < 5e-2
# Test proximal operator
beta = beta_start
function = CombinedFunction()
function.add_function(losses.LinearRegression(X, y, mean=False))
A = nesterov.l1.linear_operator_from_variables(p, penalty_start=0)
# function.add_penalty(nesterov.l1.L1(l, A=A, mu=mu_min,
# penalty_start=penalty_start))
function.add_prox(nesterov.l1.L1(l, A=A, mu=5e-5, penalty_start=0))
fista = proximal.FISTA(eps=consts.TOLERANCE, max_iter=800)
beta = fista.run(function, beta)
berr = np.linalg.norm(beta - beta_star)
# print "berr:", berr
assert berr < 5e-0
def test_smoothed_l1tv(self):
import numpy as np
from parsimony.functions import CombinedFunction
import parsimony.algorithms.proximal as proximal
import parsimony.functions as functions
import parsimony.functions.penalties as penalties
import parsimony.functions.nesterov.tv as tv
import parsimony.functions.nesterov.l1tv as l1tv
import parsimony.datasets.simulate.l1_l2_tvmu as l1_l2_tvmu
import parsimony.utils.start_vectors as start_vectors
import parsimony.datasets.simulate as simulate
np.random.seed(42)
px = 10
py = 1
pz = 1
shape = (pz, py, px)
n, p = 5, np.prod(shape)
l = 0.618
k = 0.01
g = 1.1
start_vector = start_vectors.RandomStartVector(normalise=True)
beta = start_vector.get_vector(p)
alpha = 1.0
Sigma = alpha * np.eye(p, p) \
+ (1.0 - alpha) * np.random.randn(p, p)
mean = np.zeros(p)
M = np.random.multivariate_normal(mean, Sigma, n)
e = np.random.randn(n, 1)
snr = 100.0
mu = 5e-3
A = tv.linear_operator_from_shape(shape)
# X, y, beta_star = l1_l2_tvmu.load(l=l, k=k, g=g, beta=beta, M=M, e=e,
# A=A, mu=mu, snr=snr)
funs = [
simulate.grad.L1(l),
simulate.grad.L2Squared(k),
simulate.grad.TotalVariation(g, A)
]
lr = simulate.LinearRegressionData(funs,
M,
e,
snr=snr,
intercept=False)
X, y, beta_star = lr.load(beta)
eps = 1e-8
max_iter = 810
alg = proximal.FISTA(eps=eps, max_iter=max_iter)
function = CombinedFunction()
function.add_function(
functions.losses.LinearRegression(X, y, mean=False))
function.add_penalty(penalties.L2Squared(l=k))
A = l1tv.linear_operator_from_shape(shape, p)
function.add_prox(l1tv.L1TV(l, g, A=A, mu=mu, penalty_start=0))
# A = tv.linear_operator_from_shape(shape)
# function.add_penalty(tv.TotalVariation(l=g, A=A, mu=mu,
# penalty_start=0))
# function.add_prox(penalties.L1(l=l))
beta_start = start_vector.get_vector(p)
beta = alg.run(function, beta_start)
berr = np.linalg.norm(beta - beta_star)
# print "berr:", berr
assert berr < 5e-1
f_parsimony = function.f(beta)
f_star = function.f(beta_star)
ferr = abs(f_parsimony - f_star)
# print "ferr:", ferr
assert ferr < 5e-3
def test_combo_overlapping_nonsmooth(self):
import numpy as np
from parsimony.functions import CombinedFunction
import parsimony.algorithms.proximal as proximal
import parsimony.functions as functions
import parsimony.functions.nesterov.gl as gl
import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl
import parsimony.utils.start_vectors as start_vectors
np.random.seed(42)
# Note that p must be even!
n, p = 25, 30
groups = [list(range(0, 2 * int(p / 3))), list(range(int(p / 3), p))]
weights = [1.5, 0.5]
A = gl.linear_operator_from_groups(p, groups=groups, weights=weights)
l = 0.618
k = 1.0 - l
g = 2.718
start_vector = start_vectors.RandomStartVector(normalise=True)
beta = start_vector.get_vector(p)
alpha = 1.0
Sigma = alpha * np.eye(p, p) \
+ (1.0 - alpha) * np.random.randn(p, p)
mean = np.zeros(p)
M = np.random.multivariate_normal(mean, Sigma, n)
e = np.random.randn(n, 1)
snr = 100.0
X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr)
eps = 1e-8
max_iter = 10000
beta_start = start_vector.get_vector(p)
mus = [5e-0, 5e-2, 5e-4, 5e-6, 5e-8]
fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus))
beta_parsimony = beta_start
for mu in mus:
# function = functions.LinearRegressionL1L2GL(X, y, l, k, g,
# A=A, mu=mu,
# penalty_start=0)
function = CombinedFunction()
function.add_function(
functions.losses.LinearRegression(X, y, mean=False))
function.add_penalty(functions.penalties.L2Squared(l=k))
function.add_penalty(
gl.GroupLassoOverlap(l=g, A=A, mu=mu, penalty_start=0))
function.add_prox(functions.penalties.L1(l=l))
beta_parsimony = fista.run(function, beta_parsimony)
berr = np.linalg.norm(beta_parsimony - beta_star)
# print berr
assert berr < 5e-3
f_parsimony = function.f(beta_parsimony)
f_star = function.f(beta_star)
# print abs(f_parsimony - f_star)
assert abs(f_parsimony - f_star) < 5e-6
def test_nonoverlapping_nonsmooth(self):
# Spams: http://spams-devel.gforge.inria.fr/doc-python/doc_spams.pdf
import numpy as np
from parsimony.functions import CombinedFunction
import parsimony.algorithms.proximal as proximal
import parsimony.functions as functions
import parsimony.functions.nesterov.gl as gl
import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl
import parsimony.utils.start_vectors as start_vectors
np.random.seed(42)
# Note that p must be even!
n, p = 25, 20
groups = [list(range(0, int(p / 2))), list(range(int(p / 2), p))]
# weights = [1.5, 0.5]
A = gl.linear_operator_from_groups(p,
groups=groups) # , weights=weights)
l = 0.0
k = 0.0
g = 1.0
start_vector = start_vectors.RandomStartVector(normalise=True)
beta = start_vector.get_vector(p)
alpha = 1.0
Sigma = alpha * np.eye(p, p) \
+ (1.0 - alpha) * np.random.randn(p, p)
mean = np.zeros(p)
M = np.random.multivariate_normal(mean, Sigma, n)
e = np.random.randn(n, 1)
snr = 100.0
X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr)
eps = 1e-8
max_iter = 8500
beta_start = start_vector.get_vector(p)
mus = [5e-2, 5e-4, 5e-6, 5e-8]
fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus))
beta_parsimony = beta_start
for mu in mus:
# function = functions.LinearRegressionL1L2GL(X, y, l, k, g,
# A=A, mu=mu,
# penalty_start=0)
function = CombinedFunction()
function.add_function(
functions.losses.LinearRegression(X, y, mean=False))
function.add_penalty(
gl.GroupLassoOverlap(l=g, A=A, mu=mu, penalty_start=0))
beta_parsimony = fista.run(function, beta_parsimony)
try:
import spams
params = {
"loss":
"square",
"regul":
"group-lasso-l2",
"groups":
np.array([1] * (int(p / 2)) + [2] * (int(p / 2)),
dtype=np.int32),
"lambda1":
g,
"max_it":
max_iter,
"tol":
eps,
"ista":
False,
"numThreads":
-1,
}
beta_spams, optim_info = \
spams.fistaFlat(Y=np.asfortranarray(y),
X=np.asfortranarray(X),
W0=np.asfortranarray(beta_start),
return_optim_info=True,
**params)
except ImportError:
beta_spams = np.asarray(
[[14.01111427], [35.56508563], [27.38245962], [22.39716553],
[5.835744940], [5.841502910], [2.172209350], [32.40227785],
[22.48364756], [26.48822401], [0.770391500], [36.28288883],
[31.14118214], [7.938279340], [6.800713150], [6.862914540],
[11.38161678], [19.63087584], [16.15855845], [10.89356615]])
berr = np.linalg.norm(beta_parsimony - beta_spams)
# print berr
assert berr < 5e-2
f_parsimony = function.f(beta_parsimony)
f_spams = function.f(beta_spams)
ferr = abs(f_parsimony - f_spams)
# print ferr
assert ferr < 5e-6
def test_nonoverlapping_smooth(self):
# Spams: http://spams-devel.gforge.inria.fr/doc-python/doc_spams.pdf
import numpy as np
from parsimony.functions import CombinedFunction
import parsimony.algorithms.proximal as proximal
import parsimony.functions as functions
import parsimony.functions.nesterov.gl as gl
import parsimony.datasets.simulate.l1_l2_glmu as l1_l2_glmu
import parsimony.utils.start_vectors as start_vectors
np.random.seed(42)
# Note that p must be even!
n, p = 25, 20
groups = [list(range(0, int(p / 2))), list(range(int(p / 2), p))]
# weights = [1.5, 0.5]
A = gl.linear_operator_from_groups(p,
groups=groups) # , weights=weights)
l = 0.0
k = 0.0
g = 0.9
start_vector = start_vectors.RandomStartVector(normalise=True)
beta = start_vector.get_vector(p)
alpha = 1.0
Sigma = alpha * np.eye(p, p) \
+ (1.0 - alpha) * np.random.randn(p, p)
mean = np.zeros(p)
M = np.random.multivariate_normal(mean, Sigma, n)
e = np.random.randn(n, 1)
snr = 100.0
mu_min = 5e-8
X, y, beta_star = l1_l2_glmu.load(l,
k,
g,
beta,
M,
e,
A,
mu=mu_min,
snr=snr)
eps = 1e-8
max_iter = 18000
beta_start = start_vector.get_vector(p)
mus = [5e-0, 5e-2, 5e-4, 5e-6, 5e-8]
fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus))
beta_parsimony = beta_start
for mu in mus:
# function = functions.LinearRegressionL1L2GL(X, y, l, k, g,
# A=A, mu=mu,
# penalty_start=0)
function = CombinedFunction()
function.add_function(
functions.losses.LinearRegression(X, y, mean=False))
function.add_penalty(
gl.GroupLassoOverlap(l=g, A=A, mu=mu, penalty_start=0))
beta_parsimony = fista.run(function, beta_parsimony)
try:
import spams
params = {
"loss":
"square",
"regul":
"group-lasso-l2",
"groups":
np.array([1] * (int(p / 2)) + [2] * (int(p / 2)),
dtype=np.int32),
"lambda1":
g,
"max_it":
max_iter,
"tol":
eps,
"ista":
False,
"numThreads":
-1,
}
beta_spams, optim_info = \
spams.fistaFlat(Y=np.asfortranarray(y),
X=np.asfortranarray(X),
W0=np.asfortranarray(beta_start),
return_optim_info=True,
**params)
# print beta_spams
except ImportError:
beta_spams = np.asarray(
[[15.56784201], [39.51679274], [30.42583205], [24.8816362],
[6.48671072], [6.48350546], [2.41477318], [36.00285723],
[24.98522184], [29.43128643], [0.85520539], [40.31463542],
[34.60084146], [8.82322513], [7.55741642], [7.62364398],
[12.64594707], [21.81113869], [17.95400007], [12.10507338]])
berr = np.linalg.norm(beta_parsimony - beta_spams)
# print berr
assert berr < 5e-3
f_parsimony = function.f(beta_parsimony)
f_spams = function.f(beta_spams)
ferr = abs(f_parsimony - f_spams)
# print ferr
assert ferr < 5e-6
def run(self, X, start_vector=None):
"""Find the right-singular vector of the given matrix.
Parameters
----------
X : Numpy array. The matrix to decompose.
start_vector : BaseStartVector. A start vector generator. Default is
to use a random start vector.
"""
if self.info_requested(utils.Info.ok):
self.info_set(utils.Info.ok, False)
if self.info_requested(utils.Info.time):
_t = utils.time()
if start_vector is None:
start_vector = start_vectors.RandomStartVector(normalise=True)
v0 = start_vector.get_vector(np.min(X.shape))
arpack_failed = False
try:
try:
[_, _, v] = sparse_linalg.svds(X,
k=1,
v0=v0,
tol=self.eps,
maxiter=self.max_iter,
return_singular_vectors=True)
except TypeError: # For scipy 0.9.0.
[_, _, v] = sparse_linalg.svds(X, k=1, tol=self.eps)
v = v.T
if self.info_requested(utils.Info.converged):
self.info_set(utils.Info.converged, True)
except ArpackNoConvergence:
arpack_failed = True
if arpack_failed: # Use the power method if this happens.
M, N = X.shape
if M < 80 and N < 80: # Very arbitrary threshold from one computer
_, _, V = scipy.linalg.svd(X, full_matrices=True)
v = V[[0], :].T
elif M < N:
K = np.dot(X, X.T)
t = v0
for it in xrange(self.max_iter):
t_ = t
t = np.dot(K, t_)
t *= 1.0 / maths.norm(t)
if maths.norm(t_ - t) / maths.norm(t) < self.eps:
break
v = np.dot(X.T, t)
v *= 1.0 / maths.norm(v)
else:
K = np.dot(X.T, X)
v = v0
for it in xrange(self.max_iter):
v_ = v
v = np.dot(K, v_)
v *= 1.0 / maths.norm(v)
if maths.norm(v_ - v) / maths.norm(v) < self.eps:
break
if self.info_requested(utils.Info.time):
self.info_set(utils.Info.time, utils.time() - _t)
if self.info_requested(utils.Info.func_val):
_f = maths.norm(np.dot(X, v)) # Largest singular value.
self.info_set(utils.Info.func_val, _f)
if self.info_requested(utils.Info.ok):
self.info_set(utils.Info.ok, True)
return utils.direct_vector(v)
range(int(2.0 * p / 3.0), p)
])
snr = 20.0
mu = 1e-6
#X, y, beta_star = simulated.l1_l2_tvmu.load(l, k, g, beta_star, M, e, Atv, mu, snr)
#X, y, beta_star = simulated.l1_l2_tv.load(l, k, g, beta_star, M, e, Atv, snr)
#X, y, beta_star = simulated.l1_l2_glmu.load(l, k, g, beta_star, M, e, Agl, mu, snr)
X, y, beta_star = simulated.l1_l2_gl.load(l, k, g, beta_star, M, e, Agl, snr)
f = []
fmu = []
b = []
start_vector = start_vectors.RandomStartVector()
errs = [-0.1, -0.08, -0.06, -0.04, -0.02, 0.0, 0.02, 0.04, 0.06, 0.08, 0.1]
for er in errs:
# function = functions.RR_L1_TV(X, y, k, l, g + er, A=Atv, mu=mu)
function = functions.RR_L1_GL(X, y, k, l, g + er, A=Agl, mu=mu)
beta = start_vector.get_vector(X.shape[1])
start_vector = start_vectors.IdentityStartVector(beta)
# conts = 10
# algorithm = algorithms.StaticCONESTA(mu_start=mu * 2.0 ** (conts + 6),
# output=True,
# continuations=conts, max_iter=100000)
algorithm = algorithms.FISTA(output=True, max_iter=200000)
def test_smoothed(self):
import numpy as np
from parsimony.functions import CombinedFunction
import parsimony.algorithms.proximal as proximal
import parsimony.functions.losses as losses
import parsimony.functions.nesterov as nesterov
import parsimony.utils.start_vectors as start_vectors
import parsimony.datasets.simulate.l1_l2_tv as l1_l2_tv
start_vector = start_vectors.RandomStartVector(normalise=True)
np.random.seed(42)
n, p = 75, 100
penalty_start = 0
alpha = 0.9
V = np.random.randn(p, p)
Sigma = alpha * np.eye(p, p) \
+ (1.0 - alpha) * np.dot(V.T, V)
mean = np.zeros(p)
M = np.random.multivariate_normal(mean, Sigma, n)
e = np.random.randn(n, 1)
beta = start_vector.get_vector(p)
beta[np.abs(beta) < 0.1] = 0.0
l = 0.618
k = 0.0
g = 0.0
mu_min = 0.001 # consts.TOLERANCE
A = np.eye(p)
A = [A, A, A]
snr = 100.0
X, y, beta_star = l1_l2_tv.load(l, k, g, beta, M, e, A, snr=snr)
function = CombinedFunction()
function.add_loss(losses.LinearRegression(X, y, mean=False))
A = nesterov.l1.linear_operator_from_variables(
p, penalty_start=penalty_start)
function.add_penalty(
nesterov.l1.L1(l, A=A, mu=mu_min, penalty_start=penalty_start))
# function.add_prox(penalties.L1(l, penalty_start=penalty_start))
fista = proximal.FISTA(eps=mu_min, max_iter=23500)
beta = fista.run(function, beta)
berr = np.linalg.norm(beta - beta_star)
# print "berr:", berr
# assert berr < 5
assert_less(berr, 5.0, "The found regression vector is not correct.")
# Test proximal operator
function = CombinedFunction()
function.add_loss(losses.LinearRegression(X, y, mean=False))
A = nesterov.l1.linear_operator_from_variables(
p, penalty_start=penalty_start)
function.add_prox(
nesterov.l1.L1(l, A=A, mu=mu_min, penalty_start=penalty_start))
fista = proximal.FISTA(eps=mu_min, max_iter=20000)
beta = fista.run(function, beta)
berr = np.linalg.norm(beta - beta_star)
# print "berr:", berr
# assert berr < 0.1
assert_less(berr, 0.1, "The found regression vector is not correct.")
X= np.load('/neurospin/brainomics/2014_pca_struct/synthetic_data/data_100_100/data.npy')
SHAPE = (100, 100, 1)
# a, l1, l2, tv penalties
global_pen = 0.01
tv_ratio = 0.5#1e-05
l1_ratio = 0.5
ltv = global_pen * tv_ratio
ll1 = l1_ratio * global_pen * (1 - tv_ratio)
ll2 = (1 - l1_ratio) * global_pen * (1 - tv_ratio)
assert(np.allclose(ll1 + ll2 + ltv, global_pen))
Atv = nesterov_tv.A_from_shape(SHAPE)
start_vector=start_vectors.RandomStartVector(seed=42)
##############################################################################
snapshot = AlgorithmSnapshot('/neurospin/brainomics/2014_pca_struct/synthetic_data/data_100_100_bis/').save_nipals
t0 = utils.time_cpu()
mod = pca_tv.PCA_L1_L2_TV(n_components=3,
l1=ll1, l2=ll2, ltv=ltv,
Atv=Atv,
criterion="frobenius",
eps=1e-4,
max_iter=100,
inner_max_iter=int(1e4),
output=True,start_vector=start_vector,callback=snapshot)
###############################################################################
## Models
###############################################################################
MODELS = collections.OrderedDict()
N_COMP = 3
im_shape = (100,100)
Atv = parsimony.functions.nesterov.tv.A_from_shape(im_shape)
# l2, l1, tv penalties
global_pen=0.01
l1_ratio=0.5
tv_ratio=0.5
ll1, ll2, ltv = compute_coefs_from_ratios(global_pen,tv_ratio,l1_ratio)
start_vector = start_vectors.RandomStartVector(seed=24)
MODELS["SparsePCA"] = \
sklearn.decomposition.SparsePCA(n_components=N_COMP,alpha=1)
MODELS["ElasticNetPCA"] = \
pca_tv.PCA_L1_L2_TV(n_components=N_COMP,
l1=ll1, l2=ll2, ltv=1e-6,
Atv=Atv,
criterion="frobenius",
eps=1e-6,
max_iter=100,
inner_max_iter=int(1e4),
output=False,start_vector=start_vector)
# Load true weights
if has_data:
WEIGHTS_TRUTH = np.load(weights_filename(shape, n_samples))
# Ensure that train dataset is balanced
tr = np.hstack([np.where(y.ravel() == 1)[0][:int(n_train / 2)],
np.where(y.ravel() == 0)[0][:int(n_train / 2)]])
te = np.setdiff1d(np.arange(y.shape[0]), tr)
X = X3d.reshape((n_samples, np.prod(beta3d.shape)))
Xtr = X[tr, :]
ytr = y[tr]
Xte = X[te, :]
yte = y[te]
beta_start = start_vectors.RandomStartVector().get_vector(Xtr.shape[1])
# check that ytr is balanced
#assert ytr.sum() / ytr.shape[0] == 0.5
#assert yte.sum() / yte.shape[0] == 0.53500000000000003
# Dataset with intercept
Xtr_i = np.c_[np.ones((Xtr.shape[0], 1)), Xtr]
Xte_i = np.c_[np.ones((Xte.shape[0], 1)), Xte]
beta_start_i = start_vectors.RandomStartVector().get_vector(Xtr_i.shape[1])
# global penalty
alpha = l1_max_logistic_loss(Xtr, ytr)
from parsimony.algorithms.utils import Info
info = [Info.converged,
def test_combo_smooth(self):
from parsimony.functions import CombinedFunction
import parsimony.algorithms.proximal as proximal
import parsimony.functions as functions
import parsimony.functions.nesterov.tv as tv
import parsimony.datasets.simulate.l1_l2_tvmu as l1_l2_tvmu
import parsimony.utils.start_vectors as start_vectors
np.random.seed(42)
px = 4
py = 4
pz = 4
shape = (pz, py, px)
n, p = 50, np.prod(shape)
l = 0.618
k = 1.0 - l
g = 1.1
start_vector = start_vectors.RandomStartVector(normalise=True)
beta = start_vector.get_vector(p)
alpha = 1.0
Sigma = alpha * np.eye(p, p) \
+ (1.0 - alpha) * np.random.randn(p, p)
mean = np.zeros(p)
M = np.random.multivariate_normal(mean, Sigma, n)
e = np.random.randn(n, 1)
snr = 100.0
A = tv.linear_operator_from_shape(shape)
mu_min = 5e-8
X, y, beta_star = l1_l2_tvmu.load(l=l, k=k, g=g, beta=beta, M=M, e=e,
A=A, mu=mu_min, snr=snr)
eps = 1e-8
max_iter = 5300
beta_start = start_vector.get_vector(p)
mus = [5e-2, 5e-4, 5e-6, 5e-8]
fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus))
beta_parsimony = beta_start
for mu in mus:
# function = functions.LinearRegressionL1L2GL(X, y, l, k, g,
# A=A, mu=mu,
# penalty_start=0)
function = CombinedFunction()
function.add_function(functions.losses.LinearRegression(X, y,
mean=False))
function.add_penalty(tv.TotalVariation(l=g, A=A, mu=mu,
penalty_start=0))
function.add_penalty(functions.penalties.L2Squared(l=k))
function.add_prox(functions.penalties.L1(l=l))
beta_parsimony = fista.run(function, beta_parsimony)
berr = np.linalg.norm(beta_parsimony - beta_star)
# print "berr:", berr
assert berr < 5e-3
f_parsimony = function.f(beta_parsimony)
f_star = function.f(beta_star)
ferr = abs(f_parsimony - f_star)
# print "ferr:", ferr
assert ferr < 5e-5
def test_smooth_1D_l2(self):
from parsimony.functions import CombinedFunction
import parsimony.functions as functions
import parsimony.functions.nesterov.grouptv as grouptv
import parsimony.datasets.simulate.l1_l2_grouptvmu as l1_l2_grouptvmu
import parsimony.utils.start_vectors as start_vectors
np.random.seed(1337)
n, p = 10, 15
shape = (1, 1, p)
l = 0.0
k = 0.1 # Must have some regularisation for all variables.
g = 0.9
start_vector = start_vectors.RandomStartVector(normalise=True)
beta = start_vector.get_vector(p)
rects = [[(0, 5)], [(4, 10)], [(13, 15)]]
# 0 [ 5 ] 0
# 1 [ 5 ] 0
# 2 [ 5 ] 0
# 3 [ 5 ] 0
# 4 [ 4 ] 0 / 1
beta[:5, :] = 5.0 # 5 [ 3 ] 1
beta[4, :] = 4.0 # 6 [ 3 ] 1
beta[5:10, :] = 3.0 # 7 [ 3 ] 1
beta[13:15, :] = 7.0 # 8 [ 3 ] 1
# 9 [ 3 ] 1
# 0 [ x ] -
# 1 [ x ] -
# 2 [ x ] -
# 3 [ 7 ] 2
# 4 [ 7 ] 2
alpha = 1.0
Sigma = alpha * np.eye(p, p) \
+ (1.0 - alpha) * np.random.randn(p, p)
mean = np.zeros(p)
M = np.random.multivariate_normal(mean, Sigma, n)
e = np.random.randn(n, 1)
snr = 100.0
A = grouptv.linear_operator_from_rects(rects, shape)
mu_min = 5e-8
X, y, beta_star = l1_l2_grouptvmu.load(l=l,
k=k,
g=g,
beta=beta,
M=M,
e=e,
A=A,
mu=mu_min,
snr=snr)
eps = 1e-5
max_iter = 12000
beta_start = start_vector.get_vector(p)
mus = [5e-2, 5e-4, 5e-6, 5e-8]
fista = FISTA(eps=eps, max_iter=max_iter / len(mus))
beta_parsimony = beta_start
for mu in mus:
function = CombinedFunction()
function.add_function(
functions.losses.LinearRegression(X, y, mean=False))
function.add_penalty(
grouptv.GroupTotalVariation(l=g, A=A, mu=mu, penalty_start=0))
function.add_penalty(
functions.penalties.L2Squared(l=k, penalty_start=0))
beta_parsimony = fista.run(function, beta_parsimony)
berr = np.linalg.norm(beta_parsimony - beta_star)
# print "berr:", berr
assert berr < 5e-2
f_parsimony = function.f(beta_parsimony)
f_star = function.f(beta_star)
ferr = abs(f_parsimony - f_star)
# print "ferr:", ferr
assert ferr < 5e-5
def run(self, X, start_vector=None):
"""Find the right-singular vector of the given sparse matrix.
Parameters
----------
X : Scipy sparse array. The sparse matrix to decompose.
start_vector : BaseStartVector. A start vector generator. Default is
to use a random start vector.
"""
if self.info_requested(utils.Info.ok):
self.info_set(utils.Info.ok, False)
if self.info_requested(utils.Info.time):
_t = utils.time()
if self.info_requested(utils.Info.converged):
self.info_set(utils.Info.converged, False)
if start_vector is None:
start_vector = start_vectors.RandomStartVector(normalise=True)
v0 = start_vector.get_vector(np.min(X.shape))
# determine when to use power method or scipy_sparse
use_power = True if X.shape[1] >= 10**3 else False
if not use_power:
try:
if not sp.sparse.issparse(X):
X = sp.sparse.csr_matrix(X)
try:
[_, _,
v] = sparse_linalg.svds(X,
k=1,
v0=v0,
tol=self.eps,
maxiter=self.max_iter,
return_singular_vectors=True)
except TypeError: # For scipy 0.9.0.
[_, _, v] = sparse_linalg.svds(X, k=1, tol=self.eps)
v = v.T
if self.info_requested(utils.Info.converged):
self.info_set(utils.Info.converged, True)
except ArpackNoConvergence:
use_power = True
if use_power: # Use the power method if scipy failed or if determined.
M, N = X.shape
if M < N:
K = X.dot(X.T)
t = v0
for it in xrange(self.max_iter):
t_ = t
t = K.dot(t_)
t *= 1.0 / maths.norm(t)
crit = float(maths.norm(t_ - t)) / float(maths.norm(t))
if crit < consts.TOLERANCE:
if self.info_requested(utils.Info.converged):
self.info_set(utils.Info.converged, True)
break
v = X.T.dot(t)
v *= 1.0 / maths.norm(v)
else:
K = X.T.dot(X)
v = v0
for it in xrange(self.max_iter):
v_ = v
v = K.dot(v_)
v *= 1.0 / maths.norm(v)
crit = float(maths.norm(v_ - v)) / float(maths.norm(v))
if crit < consts.TOLERANCE:
if self.info_requested(utils.Info.converged):
self.info_set(utils.Info.converged, True)
break
if self.info_requested(utils.Info.time):
self.info_set(utils.Info.time, utils.time() - _t)
if self.info_requested(utils.Info.func_val):
_f = maths.norm(X.dot(v)) # Largest singular value.
self.info_set(utils.Info.func_val, _f)
if self.info_requested(utils.Info.ok):
self.info_set(utils.Info.ok, True)
return utils.direct_vector(v)
