def __init__(self,
Q,
X,
y,
feature_weights,
ridge_term=None,
randomizer_scale=None,
perturb=None):
r"""
Create a new post-selection object for the LASSO problem
Parameters
----------
loglike : `regreg.smooth.glm.glm`
A (negative) log-likelihood as implemented in `regreg`.
feature_weights : np.ndarray
Feature weights for L-1 penalty. If a float,
it is brodcast to all features.
ridge_term : float
How big a ridge term to add?
randomizer_scale : float
Scale for IID components of randomization.
perturb : np.ndarray
Random perturbation subtracted as a linear
term in the objective function.
"""
(self.Q,
self.X,
self.y) = (Q, X, y)
self.loss = rr.quadratic_loss(Q.shape[0], Q=Q)
n, p = X.shape
self.nfeature = p
if np.asarray(feature_weights).shape == ():
feature_weights = np.ones(loglike.shape) * feature_weights
self.feature_weights = np.asarray(feature_weights)
mean_diag = np.diag(Q).mean()
if ridge_term is None:
ridge_term = np.std(y) * np.sqrt(mean_diag) / np.sqrt(n - 1)
if randomizer_scale is None:
randomizer_scale = np.sqrt(mean_diag) * 0.5 * np.std(y) * np.sqrt(n / (n - 1.))
self.randomizer = randomization.isotropic_gaussian((p,), randomizer_scale)
self.ridge_term = ridge_term
self.penalty = rr.weighted_l1norm(self.feature_weights, lagrange=1.)
self._initial_omega = perturb # random perturbation
def test_logistic_counts():
"""
Test the equivalence of binary/count specification in logistic_loglike
"""
#Form the count version of the problem
trials = np.random.binomial(5,0.5,100)+1
successes = np.random.binomial(trials,0.5,len(trials))
n = len(successes)
p = 2*n
X = np.random.normal(0,1,n*p).reshape((n,p))
loss = rr.logistic_loglike.linear(X, successes=successes, trials=trials)
penalty = rr.quadratic_loss(p, coef=1.)
prob1 = rr.container(loss, penalty)
solver1 = rr.FISTA(prob1)
solver1.fit()
solution1 = solver1.composite.coefs
#Form the binary version of the problem
Ynew = []
Xnew = []
for i, (s,n) in enumerate(zip(successes,trials)):
Ynew.append([1]*s + [0]*(n-s))
for j in range(n):
Xnew.append(X[i,:])
Ynew = np.hstack(Ynew)
Xnew = np.vstack(Xnew)
loss = rr.logistic_loglike.linear(Xnew, successes=Ynew)
penalty = rr.quadratic_loss(p, coef=1.)
prob2 = rr.container(loss, penalty)
solver2 = rr.FISTA(prob2)
solver2.fit()
solution2 = solver2.composite.coefs
npt.assert_array_almost_equal(solution1, solution2, 3)
def test_logistic_counts():
"""
Test the equivalence of binary/count specification in logistic_loglike
"""
#Form the count version of the problem
trials = np.random.binomial(5, 0.5, 100) + 1
successes = np.random.binomial(trials, 0.5, len(trials))
n = len(successes)
p = 2 * n
X = np.random.normal(0, 1, n * p).reshape((n, p))
loss = rr.logistic_loglike.linear(X, successes=successes, trials=trials)
penalty = rr.quadratic_loss(p, coef=1.)
prob1 = rr.container(loss, penalty)
solver1 = rr.FISTA(prob1)
solver1.fit()
solution1 = solver1.composite.coefs
#Form the binary version of the problem
Ynew = []
Xnew = []
for i, (s, n) in enumerate(zip(successes, trials)):
Ynew.append([1] * s + [0] * (n - s))
for j in range(n):
Xnew.append(X[i, :])
Ynew = np.hstack(Ynew)
Xnew = np.vstack(Xnew)
loss = rr.logistic_loglike.linear(Xnew, successes=Ynew)
penalty = rr.quadratic_loss(p, coef=1.)
prob2 = rr.container(loss, penalty)
solver2 = rr.FISTA(prob2)
solver2.fit()
solution2 = solver2.composite.coefs
npt.assert_array_almost_equal(solution1, solution2, 3)
def meta_algorithm(XTX, XTXi, dispersion, lam, sampler):
p = XTX.shape[0]
success = np.zeros(p)
loss = rr.quadratic_loss((p, ), Q=XTX)
pen = rr.l1norm(p, lagrange=lam)
scale = 0.5
noisy_S = sampler(scale=scale)
soln = XTXi.dot(noisy_S)
solnZ = soln / (np.sqrt(np.diag(XTXi)) * np.sqrt(dispersion))
return set(np.nonzero(np.fabs(solnZ) > 2.1)[0])
def solve_problem(Qbeta_bar, Q, lagrange, initial=None):
p = Qbeta_bar.shape[0]
loss = rr.quadratic_loss(
(p, ), Q=Q, quadratic=rr.identity_quadratic(0, 0, -Qbeta_bar, 0))
lagrange = np.asarray(lagrange)
if lagrange.shape in [(), (1, )]:
lagrange = np.ones(p) * lagrange
pen = rr.weighted_l1norm(lagrange, lagrange=1.)
problem = rr.simple_problem(loss, pen)
if initial is not None:
problem.coefs[:] = initial
soln = problem.solve(tol=1.e12, min_its=500)
return soln
def meta_algorithm(X, XTXi, resid, lam, sampler):
p = XTX.shape[0]
success = np.zeros(p)
loss = rr.quadratic_loss((p, ), Q=XTX)
pen = rr.l1norm(p, lagrange=lam)
scale = 0.
noisy_S = sampler(scale=scale)
loss.quadratic = rr.identity_quadratic(0, 0, -noisy_S, 0)
problem = rr.simple_problem(loss, pen)
soln = problem.solve(max_its=100, tol=1.e-10)
success += soln != 0
return set(np.nonzero(success)[0])
def algorithm(lam, X, y):
n, p = X.shape
success = np.zeros(p)
loss = rr.quadratic_loss((p,), Q=X.T.dot(X))
pen = rr.l1norm(p, lagrange=lam)
S = -X.T.dot(y)
loss.quadratic = rr.identity_quadratic(0, 0, S, 0)
problem = rr.simple_problem(loss, pen)
soln = problem.solve(max_its=100, tol=1.e-10)
success += soln != 0
return set(np.nonzero(success)[0])
def meta_algorithm(XTX, XTXi, dispersion, lam, sampler):
p = XTX.shape[0]
success = np.zeros(p)
loss = rr.quadratic_loss((p, ), Q=XTX)
pen = rr.l1norm(p, lagrange=lam)
scale = 0.
noisy_S = sampler(scale=scale)
soln = XTXi.dot(noisy_S)
solnZ = soln / (np.sqrt(np.diag(XTXi)) * np.sqrt(dispersion))
pval = ndist.cdf(solnZ)
pval = 2 * np.minimum(pval, 1 - pval)
return set(BHfilter(pval, q=0.2))
def _find_row_approx_inverse(Sigma,
j,
delta,
solve_args={
'min_its': 100,
'tol': 1.e-6,
'max_its': 500
}):
"""
Find an approximation of j-th row of inverse of Sigma.
Solves the problem
.. math::
\text{min}_{\theta} \frac{1}{2} \theta^TS\theta
subject to $\|\Sigma \hat{\theta} - e_j\|_{\infty} \leq \delta$ with
$e_j$ the $j$-th elementary basis vector and `S` as $\Sigma$,
and `delta` as $\delta$.
Described in Table 1, display (4) of https://arxiv.org/pdf/1306.3171.pdf
"""
p = Sigma.shape[0]
elem_basis = np.zeros(p, np.float)
elem_basis[j] = 1.
loss = quadratic_loss(p, Q=Sigma)
penalty = l1norm(p, lagrange=delta)
iq = identity_quadratic(0, 0, elem_basis, 0)
problem = simple_problem(loss, penalty)
dual_soln = problem.solve(iq, **solve_args)
soln = -dual_soln
# check feasibility -- if it fails miserably
# presume delta was too small
feasibility_gap = np.fabs(Sigma.dot(soln) - elem_basis).max()
if feasibility_gap > (1.01) * delta:
raise ValueError(
'does not seem to be a feasible point -- try increasing delta')
return soln
def test_quadratic():
l = rr.quadratic_loss(5, coef=3., offset=np.arange(5))
l.quadratic = rr.identity_quadratic(1, np.ones(5), 2 * np.ones(5), 3.)
c1 = l.get_conjugate()
q1 = rr.identity_quadratic(3, np.arange(5), 0, 0)
q2 = q1 + l.quadratic
c2 = rr.zero(5, quadratic=q2.collapsed()).conjugate
ww = np.random.standard_normal(5)
np.testing.assert_almost_equal(c2.smooth_objective(ww, 'grad'),
c1.smooth_objective(ww, 'grad'))
np.testing.assert_almost_equal(c2.objective(ww), c1.objective(ww))
np.testing.assert_almost_equal(
c2.smooth_objective(ww, 'func') + c2.nonsmooth_objective(ww),
c1.smooth_objective(ww, 'func') + c1.nonsmooth_objective(ww))
def test_quadratic():
l = rr.quadratic_loss(5, coef=3., offset=np.arange(5))
l.quadratic = rr.identity_quadratic(1, np.ones(5), 2*np.ones(5), 3.)
c1 = l.get_conjugate()
q1 = rr.identity_quadratic(3, np.arange(5), 0, 0)
q2 = q1 + l.quadratic
c2 = rr.zero(5, quadratic=q2.collapsed()).conjugate
ww = np.random.standard_normal(5)
np.testing.assert_almost_equal(c2.smooth_objective(ww, 'grad'),
c1.smooth_objective(ww, 'grad'))
np.testing.assert_almost_equal(c2.objective(ww),
c1.objective(ww))
np.testing.assert_almost_equal(c2.smooth_objective(ww, 'func') +
c2.nonsmooth_objective(ww),
c1.smooth_objective(ww, 'func') +
c1.nonsmooth_objective(ww))
