def test_nonnegative_positive_part(debug=False):
"""
This test verifies that using nonnegative constraint
with a linear term, with some unpenalized terms yields the same result
as using separable with constrained_positive_part and nonnegative
"""
import numpy as np
import regreg.api as rr
import regreg.atoms as rra
# N - number of data points
# P - number of columns in design == number of betas
N, P = 40, 30
# an arbitrary positive offset for data and design
offset = 2
# data
Y = np.random.normal(size=(N, )) + offset
# design - with ones as last column
X = np.ones((N, P))
X[:, :-1] = np.random.normal(size=(N, P - 1)) + offset
# coef for loss
coef = 0.5
# lagrange for penalty
lagrange = .1
# Loss function (squared difference between fitted and actual data)
loss = rr.quadratic_loss.affine(X, -Y, coef=coef)
# Penalty using nonnegative, leave the last 5 unpenalized but
# nonnegative
weights = np.ones(P) * lagrange
weights[-5:] = 0
linq = rr.identity_quadratic(0, 0, weights, 0)
penalty = rr.nonnegative(P, quadratic=linq)
# Solution
composite_form = rr.separable_problem.singleton(penalty, loss)
solver = rr.FISTA(composite_form)
solver.debug = debug
solver.fit(tol=1.0e-12, min_its=200)
coefs = solver.composite.coefs
# using the separable penalty, only penalize the first
# 25 coefficients with constrained_positive_part
penalties_s = [
rr.constrained_positive_part(25, lagrange=lagrange),
rr.nonnegative(5)
]
groups_s = [slice(0, 25), slice(25, 30)]
penalty_s = rr.separable((P, ), penalties_s, groups_s)
composite_form_s = rr.separable_problem.singleton(penalty_s, loss)
solver_s = rr.FISTA(composite_form_s)
solver_s.debug = debug
solver_s.fit(tol=1.0e-12, min_its=200)
coefs_s = solver_s.composite.coefs
nt.assert_true(
np.linalg.norm(coefs - coefs_s) / np.linalg.norm(coefs) < 1.0e-02)
def test_nonnegative_positive_part(debug=False):
"""
This test verifies that using nonnegative constraint
with a linear term, with some unpenalized terms yields the same result
as using separable with constrained_positive_part and nonnegative
"""
import numpy as np
import regreg.api as rr
import regreg.atoms as rra
# N - number of data points
# P - number of columns in design == number of betas
N, P = 40, 30
# an arbitrary positive offset for data and design
offset = 2
# data
Y = np.random.normal(size=(N,)) + offset
# design - with ones as last column
X = np.ones((N,P))
X[:,:-1] = np.random.normal(size=(N,P-1)) + offset
# coef for loss
coef = 0.5
# lagrange for penalty
lagrange = .1
# Loss function (squared difference between fitted and actual data)
loss = rr.quadratic.affine(X, -Y, coef=coef)
# Penalty using nonnegative, leave the last 5 unpenalized but
# nonnegative
weights = np.ones(P) * lagrange
weights[-5:] = 0
linq = rr.identity_quadratic(0,0,weights,0)
penalty = rr.nonnegative(P, quadratic=linq)
# Solution
composite_form = rr.separable_problem.singleton(penalty, loss)
solver = rr.FISTA(composite_form)
solver.debug = debug
solver.fit(tol=1.0e-12, min_its=200)
coefs = solver.composite.coefs
# using the separable penalty, only penalize the first
# 25 coefficients with constrained_positive_part
penalties_s = [rr.constrained_positive_part(25, lagrange=lagrange),
rr.nonnegative(5)]
groups_s = [slice(0,25), slice(25,30)]
penalty_s = rr.separable((P,), penalties_s,
groups_s)
composite_form_s = rr.separable_problem.singleton(penalty_s, loss)
solver_s = rr.FISTA(composite_form_s)
solver_s.debug = debug
solver_s.fit(tol=1.0e-12, min_its=200)
coefs_s = solver_s.composite.coefs
nt.assert_true(np.linalg.norm(coefs - coefs_s) / np.linalg.norm(coefs) < 1.0e-02)
def example4(lambda1=10):
#Example with an initial value for backtracking
# In the previous examples you'll see a lot of "Increasing inv_step" iterations - these are trying to find an approximate Lipschitz constant in a backtracking loop.
# For your problem the Lipschitz constant is just the largest eigenvalue of X^TX, so you can precompute this with a few power iterations.
n = 100
p = 1000
X = np.random.standard_normal(n*p).reshape((n,p))
Y = 10*np.random.standard_normal(n)
v = np.random.standard_normal(p)
for i in range(10):
v = np.dot(X.T, np.dot(X,v))
norm = np.linalg.norm(v)
v /= norm
print "Approximate Lipschitz constant is", norm
loss = rr.l2normsq.affine(X,-Y,coef=1.)
sparsity = rr.l1norm(p, lagrange = lambda1)
nonnegative = rr.nonnegative(p)
problem = rr.container(loss, sparsity, nonnegative)
solver = rr.FISTA(problem)
#Give approximate Lipschitz constant to solver
solver.fit(debug=True, start_inv_step=norm)
solution = solver.composite.coefs
def example3(lambda1=10):
#Example using a smooth approximation to the non-negativity constraint
# On large problems this might be faster than using the actual constraint
n = 100
p = 1000
X = np.random.standard_normal(n*p).reshape((n,p))
Y = 10*np.random.standard_normal(n)
loss = rr.l2normsq.affine(X,-Y,coef=1.)
sparsity = rr.l1norm(p, lagrange = lambda1)
nonnegative = rr.nonnegative(p)
smooth_nonnegative = rr.smoothed_atom(nonnegative, epsilon = 1e-4)
problem = rr.container(loss, sparsity, smooth_nonnegative)
solver = rr.FISTA(problem)
solver.fit(debug=True)
solution1 = solver.composite.coefs
loss = rr.l2normsq.affine(X,-Y,coef=1.)
sparsity = rr.l1norm(p, lagrange = lambda1)
nonnegative = rr.nonnegative(p)
problem = rr.container(loss, sparsity, nonnegative)
solver = rr.FISTA(problem)
solver.fit(debug=True)
solution2 = solver.composite.coefs
pl.subplot(1,2,1)
pl.hist(solution1, bins=40)
pl.subplot(1,2,2)
pl.scatter(solution2,solution1)
pl.xlabel("Constraint")
pl.ylabel("Smooth constraint")
def example2(lambda1=10):
#Example with a non-identity X
n = 100
p = 1000
X = np.random.standard_normal(n*p).reshape((n,p))
Y = 10*np.random.standard_normal(n)
loss = rr.l2normsq.affine(X,-Y,coef=1.)
sparsity = rr.l1norm(p, lagrange = lambda1)
nonnegative = rr.nonnegative(p)
problem = rr.container(loss, sparsity, nonnegative)
solver = rr.FISTA(problem)
solver.fit(debug=True)
solution = solver.composite.coefs
def example1(lambda1=10):
#Example with X = np.identity(n)
#Try varying lambda1 to see shrinkage
n = 100
Y = 10*np.random.standard_normal(n)
loss = rr.l2normsq.shift(-Y,coef=1.)
sparsity = rr.l1norm(n, lagrange = lambda1)
nonnegative = rr.nonnegative(n)
problem = rr.container(loss, sparsity, nonnegative)
solver = rr.FISTA(problem)
solver.fit(debug=True)
solution = solver.composite.coefs
pl.plot(Y, color='red', label='Y')
pl.plot(solution, color='blue', label='beta')
pl.legend()
# Make pure fiber data to test against
test_direction = np.array([1,2,3])/np.sqrt(14.)
#print np.sum(test_direction**2)
#unit vector for test signal direction
Y = signal_1_stick(S0,bvals[1:],d,test_direction,gradients)
# Y = design[:,3]
# Set up model according to Jonathan's recipe
N, P = X.shape
# 'coef' becomes 'lagrange' in Brad's branch
loss = rr.l2normsq.affine(X,-Y, coef=0.5)
# Select up to (not including) last column to penalize
lagrange = 500
weights = np.ones(P) * lagrange
weights[-1] = 0
penalty = rr.nonnegative(P, lagrange=1, linear_term=weights)
# Neighborhood weightings
coses2 = orientations.dot(orientations.T) ** 2
nearests = np.argmax(coses2 - np.eye(P-1), axis=1)
edges = {}
for i, nn in enumerate(nearests):
if nn > i:
tup = (i, nn)
else:
tup = (nn, i)
if tup not in edges:
edges[tup] = 1
rows, cols = zip(*edges.keys())
n_edges = len(rows)
angle_D = np.zeros((n_edges, P))
angle_D[range(n_edges), rows] = 1
def test_centering_fit_inplace(debug=False):
# N - number of data points
# P - number of columns in design == number of betas
N, P = 40, 30
# an arbitrary positive offset for data and design
offset = 2
# design - with ones as last column
X = np.random.normal(size=(N, P)) + offset
L = rr.normalize(X, center=True, scale=False, inplace=True)
# X should have been normalized in place
np.testing.assert_almost_equal(np.sum(X, 0), 0)
# data
Y = np.random.normal(size=(N, )) + offset
# coef for loss
coef = 0.5
# lagrange for penalty
lagrange = .1
# Loss function (squared difference between fitted and actual data)
loss = rr.quadratic_loss.affine(L, -Y, coef=coef)
penalties = [
rr.constrained_positive_part(25, lagrange=lagrange),
rr.nonnegative(5)
]
groups = [slice(0, 25), slice(25, 30)]
penalty = rr.separable((P, ), penalties, groups)
initial = np.random.standard_normal(P)
composite_form = rr.separable_problem.fromatom(penalty, loss)
solver = rr.FISTA(composite_form)
solver.debug = debug
solver.fit(tol=1.0e-12, min_its=200)
coefs = solver.composite.coefs
# Solve the problem with X, which has been normalized in place
loss2 = rr.quadratic_loss.affine(X, -Y, coef=coef)
initial2 = np.random.standard_normal(P)
composite_form2 = rr.separable_problem.fromatom(penalty, loss2)
solver2 = rr.FISTA(composite_form2)
solver2.debug = debug
solver2.fit(tol=1.0e-12, min_its=200)
coefs2 = solver2.composite.coefs
for _ in range(10):
beta = np.random.standard_normal(P)
g1 = loss.smooth_objective(beta, mode='grad')
g2 = loss2.smooth_objective(beta, mode='grad')
np.testing.assert_almost_equal(g1, g2)
b1 = penalty.proximal(sq(1, beta, g1, 0))
b2 = penalty.proximal(sq(1, beta, g2, 0))
np.testing.assert_almost_equal(b1, b2)
f1 = composite_form.objective(beta)
f2 = composite_form2.objective(beta)
np.testing.assert_almost_equal(f1, f2)
np.testing.assert_almost_equal(composite_form.objective(coefs),
composite_form.objective(coefs2))
np.testing.assert_almost_equal(composite_form2.objective(coefs),
composite_form2.objective(coefs2))
nt.assert_true(
np.linalg.norm(coefs - coefs2) /
max(np.linalg.norm(coefs), 1) < 1.0e-04)
def test_centering_fit_inplace(debug=False):
# N - number of data points
# P - number of columns in design == number of betas
N, P = 40, 30
# an arbitrary positive offset for data and design
offset = 2
# design - with ones as last column
X = np.random.normal(size=(N,P)) + offset
L = rr.normalize(X, center=True, scale=False, inplace=True)
# X should have been normalized in place
np.testing.assert_almost_equal(np.sum(X, 0), 0)
# data
Y = np.random.normal(size=(N,)) + offset
# coef for loss
coef = 0.5
# lagrange for penalty
lagrange = .1
# Loss function (squared difference between fitted and actual data)
loss = rr.quadratic.affine(L, -Y, coef=coef)
penalties = [rr.constrained_positive_part(25, lagrange=lagrange),
rr.nonnegative(5)]
groups = [slice(0,25), slice(25,30)]
penalty = rr.separable((P,), penalties,
groups)
initial = np.random.standard_normal(P)
composite_form = rr.separable_problem.fromatom(penalty, loss)
solver = rr.FISTA(composite_form)
solver.debug = debug
solver.fit(tol=1.0e-12, min_its=200)
coefs = solver.composite.coefs
# Solve the problem with X, which has been normalized in place
loss2 = rr.quadratic.affine(X, -Y, coef=coef)
initial2 = np.random.standard_normal(P)
composite_form2 = rr.separable_problem.fromatom(penalty, loss2)
solver2 = rr.FISTA(composite_form2)
solver2.debug = debug
solver2.fit(tol=1.0e-12, min_its=200)
coefs2 = solver2.composite.coefs
for _ in range(10):
beta = np.random.standard_normal(P)
g1 = loss.smooth_objective(beta, mode='grad')
g2 = loss2.smooth_objective(beta, mode='grad')
np.testing.assert_almost_equal(g1, g2)
b1 = penalty.proximal(sq(1, beta, g1,0))
b2 = penalty.proximal(sq(1, beta, g2,0))
np.testing.assert_almost_equal(b1, b2)
f1 = composite_form.objective(beta)
f2 = composite_form2.objective(beta)
np.testing.assert_almost_equal(f1, f2)
np.testing.assert_almost_equal(composite_form.objective(coefs), composite_form.objective(coefs2))
np.testing.assert_almost_equal(composite_form2.objective(coefs), composite_form2.objective(coefs2))
nt.assert_true(np.linalg.norm(coefs - coefs2) / max(np.linalg.norm(coefs),1) < 1.0e-04)
def test_scaling_and_centering_intercept_fit(debug=False):
# N - number of data points
# P - number of columns in design == number of betas
N, P = 40, 30
# an arbitrary positive offset for data and design
offset = 2
# design - with ones as last column
X = np.random.normal(size=(N, P)) + 0 * offset
X2 = X - X.mean(0)[None, :]
X2 = X2 / np.std(X2, 0, ddof=1)[None, :]
X2 = np.hstack([np.ones((X2.shape[0], 1)), X2])
L = rr.normalize(X, center=True, scale=True, intercept=True)
# data
Y = np.random.normal(size=(N, )) + offset
# lagrange for penalty
lagrange = .1
# Loss function (squared difference between fitted and actual data)
loss = rr.squared_error(L, Y)
penalties = [
rr.constrained_positive_part(25, lagrange=lagrange),
rr.nonnegative(5)
]
groups = [slice(0, 25), slice(25, 30)]
penalty = rr.separable((P + 1, ), penalties, groups)
initial = np.random.standard_normal(P + 1)
composite_form = rr.separable_problem.fromatom(penalty, loss)
solver = rr.FISTA(composite_form)
solver.debug = debug
solver.fit(tol=1.0e-12, min_its=200)
coefs = solver.composite.coefs
# Solve the problem with X2
loss2 = rr.squared_error(X2, Y)
initial2 = np.random.standard_normal(P + 1)
composite_form2 = rr.separable_problem.fromatom(penalty, loss2)
solver2 = rr.FISTA(composite_form2)
solver2.debug = debug
solver2.fit(tol=1.0e-12, min_its=200)
coefs2 = solver2.composite.coefs
for _ in range(10):
beta = np.random.standard_normal(P + 1)
g1 = loss.smooth_objective(beta, mode='grad')
g2 = loss2.smooth_objective(beta, mode='grad')
np.testing.assert_almost_equal(g1, g2)
b1 = penalty.proximal(sq(1, beta, g1, 0))
b2 = penalty.proximal(sq(1, beta, g2, 0))
np.testing.assert_almost_equal(b1, b2)
f1 = composite_form.objective(beta)
f2 = composite_form2.objective(beta)
np.testing.assert_almost_equal(f1, f2)
np.testing.assert_almost_equal(composite_form.objective(coefs),
composite_form.objective(coefs2))
np.testing.assert_almost_equal(composite_form2.objective(coefs),
composite_form2.objective(coefs2))
nt.assert_true(
np.linalg.norm(coefs - coefs2) /
max(np.linalg.norm(coefs), 1) < 1.0e-04)
print('Solving L1 penalized system\n'
'\n'
' ||Ax - b|| + lam |x|_1, subject to x_i >= 0.'
'\n\n'
'Here A is reproducing kernel in sparse matrix form.\n'
'b is the measured signal and x are the coefficents in\n'
'the sparse represenation.\n\n')
A = A_new
b = rSig
# Set up the problem
loss = rr.l2normsq.affine(A, -b, coef=1.)
sparsity = rr.l1norm(n_qpnts, lagrange=lamb)
constraint = rr.nonnegative(n_qpnts)
problem = rr.container(loss, sparsity, constraint)
# Call the solver
solver = rr.FISTA(problem)
solver.fit(debug=False)
nd_coefs_l1 = solver.composite.coefs
# Cutoff those coefficients that are less than cutoff
cutoff = nd_coefs_l1.mean() + 2.5*nd_coefs_l1.std(ddof=1)
nd_coefs_l1_trim = np.where(nd_coefs_l1 > cutoff, nd_coefs_l1, 0)
# Get indices needed for sorting coefs, in reverse order.
sortedIndex = nd_coefs_l1_trim.argsort()[::-1]
# number of significant coefficients
