def test_nonoverlapping_nonsmooth(self):
# Spams: http://spams-devel.gforge.inria.fr/doc-python/doc_spams.pdf
import numpy as np
from parsimony.functions import CombinedFunction
import parsimony.algorithms.proximal as proximal
import parsimony.functions as functions
import parsimony.functions.nesterov.gl as gl
import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl
import parsimony.utils.start_vectors as start_vectors
np.random.seed(42)
# Note that p must be even!
n, p = 25, 20
groups = [range(0, p / 2), range(p / 2, p)]
# weights = [1.5, 0.5]
A = gl.A_from_groups(p, groups=groups) # , weights=weights)
l = 0.0
k = 0.0
g = 1.0
start_vector = start_vectors.RandomStartVector(normalise=True)
beta = start_vector.get_vector(p)
alpha = 1.0
Sigma = alpha * np.eye(p, p) \
+ (1.0 - alpha) * np.random.randn(p, p)
mean = np.zeros(p)
M = np.random.multivariate_normal(mean, Sigma, n)
e = np.random.randn(n, 1)
snr = 100.0
X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr)
eps = 1e-8
max_iter = 8500
beta_start = start_vector.get_vector(p)
mus = [5e-2, 5e-4, 5e-6, 5e-8]
fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus))
beta_parsimony = beta_start
for mu in mus:
# function = functions.LinearRegressionL1L2GL(X, y, l, k, g,
# A=A, mu=mu,
# penalty_start=0)
function = CombinedFunction()
function.add_function(functions.losses.LinearRegression(X, y,
mean=False))
function.add_penalty(gl.GroupLassoOverlap(l=g, A=A, mu=mu,
penalty_start=0))
beta_parsimony = fista.run(function, beta_parsimony)
try:
import spams
params = {"loss": "square",
"regul": "group-lasso-l2",
"groups": np.array([1] * (p / 2) + [2] * (p / 2),
dtype=np.int32),
"lambda1": g,
"max_it": max_iter,
"tol": eps,
"ista": False,
"numThreads": -1,
}
beta_spams, optim_info = \
spams.fistaFlat(Y=np.asfortranarray(y),
X=np.asfortranarray(X),
W0=np.asfortranarray(beta_start),
return_optim_info=True,
**params)
except ImportError:
beta_spams = np.asarray([[14.01111427],
[35.56508563],
[27.38245962],
[22.39716553],
[5.835744940],
[5.841502910],
[2.172209350],
[32.40227785],
[22.48364756],
[26.48822401],
[0.770391500],
[36.28288883],
[31.14118214],
[7.938279340],
[6.800713150],
[6.862914540],
[11.38161678],
[19.63087584],
[16.15855845],
[10.89356615]])
berr = np.linalg.norm(beta_parsimony - beta_spams)
# print berr
assert berr < 5e-2
f_parsimony = function.f(beta_parsimony)
f_spams = function.f(beta_spams)
ferr = abs(f_parsimony - f_spams)
# print ferr
assert ferr < 5e-6
def test_combo_overlapping_nonsmooth(self):
import numpy as np
from parsimony.functions import CombinedFunction
import parsimony.algorithms.proximal as proximal
import parsimony.functions as functions
import parsimony.functions.nesterov.gl as gl
import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl
import parsimony.utils.start_vectors as start_vectors
np.random.seed(42)
# Note that p must be even!
n, p = 25, 30
groups = [range(0, 2 * p / 3), range(p / 3, p)]
weights = [1.5, 0.5]
A = gl.A_from_groups(p, groups=groups, weights=weights)
l = 0.618
k = 1.0 - l
g = 2.718
start_vector = start_vectors.RandomStartVector(normalise=True)
beta = start_vector.get_vector(p)
alpha = 1.0
Sigma = alpha * np.eye(p, p) \
+ (1.0 - alpha) * np.random.randn(p, p)
mean = np.zeros(p)
M = np.random.multivariate_normal(mean, Sigma, n)
e = np.random.randn(n, 1)
snr = 100.0
X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr)
eps = 1e-8
max_iter = 10000
beta_start = start_vector.get_vector(p)
mus = [5e-0, 5e-2, 5e-4, 5e-6, 5e-8]
fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus))
beta_parsimony = beta_start
for mu in mus:
# function = functions.LinearRegressionL1L2GL(X, y, l, k, g,
# A=A, mu=mu,
# penalty_start=0)
function = CombinedFunction()
function.add_function(functions.losses.LinearRegression(X, y,
mean=False))
function.add_penalty(functions.penalties.L2Squared(l=k))
function.add_penalty(gl.GroupLassoOverlap(l=g, A=A, mu=mu,
penalty_start=0))
function.add_prox(functions.penalties.L1(l=l))
beta_parsimony = fista.run(function, beta_parsimony)
berr = np.linalg.norm(beta_parsimony - beta_star)
# print berr
assert berr < 5e-3
f_parsimony = function.f(beta_parsimony)
f_star = function.f(beta_star)
# print abs(f_parsimony - f_star)
assert abs(f_parsimony - f_star) < 5e-6
def test_DynamicCONESTA_gl(self):
import numpy as np
np.random.seed(42)
import parsimony.estimators as estimators
import parsimony.functions.nesterov.gl as gl
import parsimony.utils.start_vectors as start_vectors
import parsimony.algorithms.primaldual as primaldual
import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl
start_vector = start_vectors.RandomStartVector(normalise=True,
limits=(-1, 1))
# Note that p should be divisible by 3!
n, p = 75, 90
penalty_start = 0
groups = [range(penalty_start, 2 * p / 3), range(p / 3, p)]
weights = [1.5, 0.5]
l = 0.618
k = 1.0 - l
g = 1.618
snr = 100.0
A = gl.A_from_groups(p, groups=groups, weights=weights,
penalty_start=penalty_start)
alpha = 0.9
Sigma = alpha * np.eye(p - penalty_start,
p - penalty_start) \
+ (1.0 - alpha) * np.random.randn(p - penalty_start,
p - penalty_start)
mean = np.zeros(p - penalty_start)
M = np.random.multivariate_normal(mean, Sigma, n)
if penalty_start > 0:
M = np.hstack((np.ones((n, 1)), M))
e = np.random.randn(n, 1)
while np.min(np.abs(np.dot(M.T, e))) < 1.0 / np.sqrt(n) \
or np.max(np.abs(np.dot(M.T, e))) > n:
e = np.random.randn(n, 1)
beta = start_vector.get_vector(p)
beta = np.sort(beta, axis=0)
beta[np.abs(beta) < 0.05] = 0.0
if penalty_start > 0:
beta[0, 0] = 2.7182818
X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr,
intercept=penalty_start > 0)
eps = 1e-8
max_iter = 4200
mu = None
dynamic = estimators.LinearRegressionL1L2GL(l, k, g, A, mu=mu,
algorithm=primaldual.DynamicCONESTA(),
algorithm_params=dict(eps=eps,
max_iter=max_iter),
penalty_start=penalty_start,
mean=False)
dynamic.fit(X, y)
err = dynamic.score(X, y)
# print "err :", err
beta_dynamic = dynamic.beta
dynamic.beta = beta_star
err_star = dynamic.score(X, y)
# print "err*:", err_star
serr = abs(err - err_star)
# print "score diff:", serr
assert_less(serr, 5e-3,
msg="The algorithm did not find a minimiser.")
berr = np.linalg.norm(beta_dynamic - beta_star)
# print "beta diff:", berr
assert_less(berr, 5e-3,
msg="The algorithm did not find a minimiser.")
def test_combo_overlapping_nonsmooth(self):
import numpy as np
from parsimony.functions import CombinedFunction
import parsimony.algorithms.proximal as proximal
import parsimony.functions as functions
import parsimony.functions.nesterov.gl as gl
import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl
import parsimony.utils.weights as weights
np.random.seed(42)
# Note that p must be even!
n, p = 25, 30
groups = [list(range(0, 2 * int(p / 3))), list(range(int(p / 3), p))]
group_weights = [1.5, 0.5]
A = gl.linear_operator_from_groups(p,
groups=groups,
weights=group_weights)
l = 0.618
k = 1.0 - l
g = 2.718
start_vector = weights.RandomUniformWeights(normalise=True)
beta = start_vector.get_weights(p)
alpha = 1.0
Sigma = alpha * np.eye(p, p) \
+ (1.0 - alpha) * np.random.randn(p, p)
mean = np.zeros(p)
M = np.random.multivariate_normal(mean, Sigma, n)
e = np.random.randn(n, 1)
snr = 100.0
X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr)
eps = 1e-8
max_iter = 10000
beta_start = start_vector.get_weights(p)
mus = [5e-0, 5e-2, 5e-4, 5e-6, 5e-8]
fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus))
beta_parsimony = beta_start
for mu in mus:
# function = functions.LinearRegressionL1L2GL(X, y, l, k, g,
# A=A, mu=mu,
# penalty_start=0)
function = CombinedFunction()
function.add_loss(
functions.losses.LinearRegression(X, y, mean=False))
function.add_penalty(functions.penalties.L2Squared(l=k))
function.add_penalty(
gl.GroupLassoOverlap(l=g, A=A, mu=mu, penalty_start=0))
function.add_prox(functions.penalties.L1(l=l))
beta_parsimony = fista.run(function, beta_parsimony)
berr = np.linalg.norm(beta_parsimony - beta_star)
# print berr
assert berr < 5e-3
f_parsimony = function.f(beta_parsimony)
f_star = function.f(beta_star)
# print abs(f_parsimony - f_star)
assert abs(f_parsimony - f_star) < 5e-6
def test_nonoverlapping_nonsmooth(self):
# Spams: http://spams-devel.gforge.inria.fr/doc-python/doc_spams.pdf
import numpy as np
from parsimony.functions import CombinedFunction
import parsimony.algorithms.proximal as proximal
import parsimony.functions as functions
import parsimony.functions.nesterov.gl as gl
import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl
import parsimony.utils.start_vectors as start_vectors
np.random.seed(42)
# Note that p must be even!
n, p = 25, 20
groups = [list(range(0, int(p / 2))), list(range(int(p / 2), p))]
# weights = [1.5, 0.5]
A = gl.linear_operator_from_groups(p,
groups=groups) # , weights=weights)
l = 0.0
k = 0.0
g = 1.0
start_vector = start_vectors.RandomStartVector(normalise=True)
beta = start_vector.get_vector(p)
alpha = 1.0
Sigma = alpha * np.eye(p, p) \
+ (1.0 - alpha) * np.random.randn(p, p)
mean = np.zeros(p)
M = np.random.multivariate_normal(mean, Sigma, n)
e = np.random.randn(n, 1)
snr = 100.0
X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr)
eps = 1e-8
max_iter = 8500
beta_start = start_vector.get_vector(p)
mus = [5e-2, 5e-4, 5e-6, 5e-8]
fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus))
beta_parsimony = beta_start
for mu in mus:
# function = functions.LinearRegressionL1L2GL(X, y, l, k, g,
# A=A, mu=mu,
# penalty_start=0)
function = CombinedFunction()
function.add_function(
functions.losses.LinearRegression(X, y, mean=False))
function.add_penalty(
gl.GroupLassoOverlap(l=g, A=A, mu=mu, penalty_start=0))
beta_parsimony = fista.run(function, beta_parsimony)
try:
import spams
params = {
"loss":
"square",
"regul":
"group-lasso-l2",
"groups":
np.array([1] * (int(p / 2)) + [2] * (int(p / 2)),
dtype=np.int32),
"lambda1":
g,
"max_it":
max_iter,
"tol":
eps,
"ista":
False,
"numThreads":
-1,
}
beta_spams, optim_info = \
spams.fistaFlat(Y=np.asfortranarray(y),
X=np.asfortranarray(X),
W0=np.asfortranarray(beta_start),
return_optim_info=True,
**params)
except ImportError:
beta_spams = np.asarray(
[[14.01111427], [35.56508563], [27.38245962], [22.39716553],
[5.835744940], [5.841502910], [2.172209350], [32.40227785],
[22.48364756], [26.48822401], [0.770391500], [36.28288883],
[31.14118214], [7.938279340], [6.800713150], [6.862914540],
[11.38161678], [19.63087584], [16.15855845], [10.89356615]])
berr = np.linalg.norm(beta_parsimony - beta_spams)
# print berr
assert berr < 5e-2
f_parsimony = function.f(beta_parsimony)
f_spams = function.f(beta_spams)
ferr = abs(f_parsimony - f_spams)
# print ferr
assert ferr < 5e-6
def test_overlapping_nonsmooth(self):
import numpy as np
from parsimony.functions import CombinedFunction
import parsimony.algorithms.proximal as proximal
import parsimony.functions as functions
import parsimony.functions.nesterov.gl as gl
import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl
import parsimony.utils.weights as weights
np.random.seed(42)
# Note that p should be divisible by 3!
n, p = 25, 30
groups = [list(range(0, 2 * int(p / 3))), list(range(int(p / 3), p))]
group_weights = [1.5, 0.5]
A = gl.linear_operator_from_groups(p, groups=groups,
weights=group_weights)
l = 0.0
k = 0.0
g = 1.1
start_vector = weights.RandomUniformWeights(normalise=True)
beta = start_vector.get_weights(p)
alpha = 1.0
Sigma = alpha * np.eye(p, p) \
+ (1.0 - alpha) * np.random.randn(p, p)
mean = np.zeros(p)
M = np.random.multivariate_normal(mean, Sigma, n)
e = np.random.randn(n, 1)
snr = 100.0
X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr)
eps = 1e-8
max_iter = 8000
beta_start = start_vector.get_weights(p)
mus = [5e-2, 5e-4, 5e-6, 5e-8]
fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus))
beta_parsimony = beta_start
for mu in mus:
# function = functions.LinearRegressionL1L2GL(X, y, l, k, g,
# A=A, mu=mu,
# penalty_start=0)
function = CombinedFunction()
function.add_loss(functions.losses.LinearRegression(X, y,
mean=False))
function.add_penalty(gl.GroupLassoOverlap(l=g, A=A, mu=mu,
penalty_start=0))
beta_parsimony = fista.run(function, beta_parsimony)
berr = np.linalg.norm(beta_parsimony - beta_star)
# print berr
assert berr < 0.05
f_parsimony = function.f(beta_parsimony)
f_star = function.f(beta_star)
# print(abs(f_parsimony - f_star))
assert abs(f_parsimony - f_star) < 5e-4
def test_overlapping_nonsmooth(self):
import numpy as np
from parsimony.functions import CombinedFunction
import parsimony.algorithms.proximal as proximal
import parsimony.functions as functions
import parsimony.functions.nesterov.gl as gl
import parsimony.datasets.simulate.l1_l2_gl as l1_l2_gl
import parsimony.utils.start_vectors as start_vectors
np.random.seed(42)
# Note that p should be divisible by 3!
n, p = 25, 30
groups = [range(0, 2 * p / 3), range(p / 3, p)]
weights = [1.5, 0.5]
A = gl.linear_operator_from_groups(p, groups=groups, weights=weights)
l = 0.0
k = 0.0
g = 1.1
start_vector = start_vectors.RandomStartVector(normalise=True)
beta = start_vector.get_vector(p)
alpha = 1.0
Sigma = alpha * np.eye(p, p) \
+ (1.0 - alpha) * np.random.randn(p, p)
mean = np.zeros(p)
M = np.random.multivariate_normal(mean, Sigma, n)
e = np.random.randn(n, 1)
snr = 100.0
X, y, beta_star = l1_l2_gl.load(l, k, g, beta, M, e, A, snr=snr)
eps = 1e-8
max_iter = 8000
beta_start = start_vector.get_vector(p)
mus = [5e-2, 5e-4, 5e-6, 5e-8]
fista = proximal.FISTA(eps=eps, max_iter=max_iter / len(mus))
beta_parsimony = beta_start
for mu in mus:
# function = functions.LinearRegressionL1L2GL(X, y, l, k, g,
# A=A, mu=mu,
# penalty_start=0)
function = CombinedFunction()
function.add_function(
functions.losses.LinearRegression(X, y, mean=False))
function.add_penalty(
gl.GroupLassoOverlap(l=g, A=A, mu=mu, penalty_start=0))
beta_parsimony = fista.run(function, beta_parsimony)
berr = np.linalg.norm(beta_parsimony - beta_star)
# print berr
assert berr < 0.05
f_parsimony = function.f(beta_parsimony)
f_star = function.f(beta_star)
# print abs(f_parsimony - f_star)
assert abs(f_parsimony - f_star) < 5e-5
