def test_goodness_of_fit(n=20, p=25, s=10, sigma=20.,
nsample=1000):
P = []
while True:
y = np.random.standard_normal(n) * sigma
beta = np.zeros(p)
X = np.random.standard_normal((n,p)) + 0.3 * np.random.standard_normal(n)[:,None]
X /= (X.std(0)[None,:] * np.sqrt(n))
y += np.dot(X, beta) * sigma
lam_theor = .7 * choose_lambda(X, quantile=0.9)
L = sqrt_lasso(y, X, lam_theor)
L.fit(tol=1.e-12, min_its=150, max_its=200)
pval = L.goodness_of_fit(lambda x: np.max(np.fabs(x)),
burnin=10000,
ndraw=10000)
P.append(pval)
Pa = np.array(P)
Pa = Pa[~np.isnan(Pa)]
#print (~np.isnan(np.array(Pa))).sum()
if (~np.isnan(np.array(Pa))).sum() >= nsample:
break
#print np.mean(Pa), np.std(Pa)
U = np.linspace(0,1,nsample+1)
plt.plot(U, sm.distributions.ECDF(Pa)(U))
plt.plot([0,1], [0,1])
plt.savefig("goodness_of_fit_uniform", format="pdf")
def test_gaussian_approx(n=100,p=200,s=10):
"""
using gaussian approximation for pvalues
"""
sigma = 3
y = np.random.standard_normal(n) * sigma
beta = np.zeros(p)
#beta[:s] = 8 * (2 * np.random.binomial(1, 0.5, size=(s,)) - 1)
beta[:s] = 18
X = np.random.standard_normal((n,p)) + 0.3 * np.random.standard_normal(n)[:,None]
X /= (X.std(0)[None,:] * np.sqrt(n))
y += np.dot(X, beta)
lam_theor = choose_lambda(X, quantile=0.75)
L = sqrt_lasso(y, X, lam_theor)
L.fit(tol=1.e-10, min_its=80)
P = []
P_gaussian = []
intervals = []
if L.active.shape[0] > 0:
np.testing.assert_array_less( \
np.dot(L.constraints.linear_part, L.y),
L.constraints.offset)
if set(range(s)).issubset(L.active):
P = [p[1] for p in L.active_pvalues[s:]]
P_gaussian = [p[1] for p in L.active_gaussian_pval[s:]]
intervals = [u for u in L.active_gaussian_intervals if u[0] in range(s)]
return P, P_gaussian, intervals, beta
def test_class(n=20, p=40, s=2):
y = np.random.standard_normal(n) * 1.2
beta = np.zeros(p)
beta[:s] = 5
X = np.random.standard_normal(
(n, p)) + 0.3 * np.random.standard_normal(n)[:, None]
y += np.dot(X, beta)
lam_theor = 0.7 * choose_lambda(X, quantile=0.9)
L = sqrt_lasso(y, X, lam_theor)
L.fit(tol=1.e-10, min_its=80)
P = []
if L.active.shape[0] > 0:
np.testing.assert_array_less( \
np.dot(L.constraints.linear_part, L.y),
L.constraints.offset)
nt.assert_true(L.constraints(y))
nt.assert_true(L.quasi_affine_constraints(y))
if set(range(s)).issubset(L.active):
P = [p[1] for p in L.active_pvalues[s:]]
else:
P = []
return P
def test_class_R(n=100, p=20):
y = np.random.standard_normal(n)
X = np.random.standard_normal((n,p))
lam_theor = choose_lambda(X, quantile=0.25)
L = sqrt_lasso(y,X,lam_theor)
L.fit(tol=1.e-7)
if L.active.shape[0] > 0:
np.testing.assert_array_less( \
np.dot(L.constraints.linear_part, L.y),
L.constraints.offset)
return L.active_constraints.linear_part, L.active_constraints.offset / L.sigma_E, L.R_E, L._XEinv[0]
else:
return None, None, None, None
def _generate_constraints(n=15, p=20, sigma=1):
while True:
y = np.random.standard_normal(n) * sigma
beta = np.zeros(p)
X = np.random.standard_normal((n,p)) + 0.3 * np.random.standard_normal(n)[:,None]
X /= (X.std(0)[None,:] * np.sqrt(n))
y += np.dot(X, beta) * sigma
lam_theor = 0.3 * choose_lambda(X, quantile=0.9)
L = sqrt_lasso(y, X, lam_theor)
L.fit(tol=1.e-12, min_its=150)
con = L.active_constraints
if con is not None and L.active.shape[0] >= 3:
break
con.covariance = np.identity(con.covariance.shape[0])
con.mean *= 0
return con, y, L
def test_estimate_sigma(n=200, p=400, s=10, sigma=3.):
y = np.random.standard_normal(n) * sigma
beta = np.zeros(p)
beta[:s] = 8 * (2 * np.random.binomial(1, 0.5, size=(s,)) - 1)
X = np.random.standard_normal((n,p)) + 0.3 * np.random.standard_normal(n)[:,None]
X /= (X.std(0)[None,:] * np.sqrt(n))
y += np.dot(X, beta) * sigma
lam_theor = choose_lambda(X, quantile=0.9)
L = sqrt_lasso(y, X, lam_theor)
L.fit(tol=1.e-12, min_its=150)
P = []
if L.active.shape[0] > 0:
return L.sigma_hat / sigma, L.sigma_E / sigma, L.df_E
else:
return (None,) * 3
def _generate_constraints(n=15, p=20, sigma=1):
while True:
y = np.random.standard_normal(n) * sigma
beta = np.zeros(p)
X = np.random.standard_normal(
(n, p)) + 0.3 * np.random.standard_normal(n)[:, None]
X /= (X.std(0)[None, :] * np.sqrt(n))
y += np.dot(X, beta) * sigma
lam_theor = 0.3 * choose_lambda(X, quantile=0.9)
L = sqrt_lasso(y, X, lam_theor)
L.fit(tol=1.e-12, min_its=150)
con = L.active_constraints
if con is not None and L.active.shape[0] >= 3:
break
con.covariance = np.identity(con.covariance.shape[0])
con.mean *= 0
return con, y, L
def test_class(n=20, p=40, s=2):
y = np.random.standard_normal(n) * 1.2
beta = np.zeros(p)
beta[:s] = 5
X = np.random.standard_normal((n,p)) + 0.3 * np.random.standard_normal(n)[:,None]
y += np.dot(X, beta)
lam_theor = 0.7 * choose_lambda(X, quantile=0.9)
L = sqrt_lasso(y,X,lam_theor)
L.fit(tol=1.e-10, min_its=80)
P = []
if L.active.shape[0] > 0:
np.testing.assert_array_less( \
np.dot(L.constraints.linear_part, L.y),
L.constraints.offset)
nt.assert_true(L.constraints(y))
nt.assert_true(L.quasi_affine_constraints(y))
if set(range(s)).issubset(L.active):
P = [p[1] for p in L.active_pvalues[s:]]
else:
P = []
return P
