def test_goodness_of_fit(n=20, p=25, s=10, sigma=20., nsample=1000): P = [] while True: y = np.random.standard_normal(n) * sigma beta = np.zeros(p) X = np.random.standard_normal((n,p)) + 0.3 * np.random.standard_normal(n)[:,None] X /= (X.std(0)[None,:] * np.sqrt(n)) y += np.dot(X, beta) * sigma lam_theor = .7 * choose_lambda(X, quantile=0.9) L = sqrt_lasso(y, X, lam_theor) L.fit(tol=1.e-12, min_its=150, max_its=200) pval = L.goodness_of_fit(lambda x: np.max(np.fabs(x)), burnin=10000, ndraw=10000) P.append(pval) Pa = np.array(P) Pa = Pa[~np.isnan(Pa)] #print (~np.isnan(np.array(Pa))).sum() if (~np.isnan(np.array(Pa))).sum() >= nsample: break #print np.mean(Pa), np.std(Pa) U = np.linspace(0,1,nsample+1) plt.plot(U, sm.distributions.ECDF(Pa)(U)) plt.plot([0,1], [0,1]) plt.savefig("goodness_of_fit_uniform", format="pdf")
def test_gaussian_approx(n=100,p=200,s=10): """ using gaussian approximation for pvalues """ sigma = 3 y = np.random.standard_normal(n) * sigma beta = np.zeros(p) #beta[:s] = 8 * (2 * np.random.binomial(1, 0.5, size=(s,)) - 1) beta[:s] = 18 X = np.random.standard_normal((n,p)) + 0.3 * np.random.standard_normal(n)[:,None] X /= (X.std(0)[None,:] * np.sqrt(n)) y += np.dot(X, beta) lam_theor = choose_lambda(X, quantile=0.75) L = sqrt_lasso(y, X, lam_theor) L.fit(tol=1.e-10, min_its=80) P = [] P_gaussian = [] intervals = [] if L.active.shape[0] > 0: np.testing.assert_array_less( \ np.dot(L.constraints.linear_part, L.y), L.constraints.offset) if set(range(s)).issubset(L.active): P = [p[1] for p in L.active_pvalues[s:]] P_gaussian = [p[1] for p in L.active_gaussian_pval[s:]] intervals = [u for u in L.active_gaussian_intervals if u[0] in range(s)] return P, P_gaussian, intervals, beta
def test_class(n=20, p=40, s=2): y = np.random.standard_normal(n) * 1.2 beta = np.zeros(p) beta[:s] = 5 X = np.random.standard_normal( (n, p)) + 0.3 * np.random.standard_normal(n)[:, None] y += np.dot(X, beta) lam_theor = 0.7 * choose_lambda(X, quantile=0.9) L = sqrt_lasso(y, X, lam_theor) L.fit(tol=1.e-10, min_its=80) P = [] if L.active.shape[0] > 0: np.testing.assert_array_less( \ np.dot(L.constraints.linear_part, L.y), L.constraints.offset) nt.assert_true(L.constraints(y)) nt.assert_true(L.quasi_affine_constraints(y)) if set(range(s)).issubset(L.active): P = [p[1] for p in L.active_pvalues[s:]] else: P = [] return P
def test_class_R(n=100, p=20): y = np.random.standard_normal(n) X = np.random.standard_normal((n,p)) lam_theor = choose_lambda(X, quantile=0.25) L = sqrt_lasso(y,X,lam_theor) L.fit(tol=1.e-7) if L.active.shape[0] > 0: np.testing.assert_array_less( \ np.dot(L.constraints.linear_part, L.y), L.constraints.offset) return L.active_constraints.linear_part, L.active_constraints.offset / L.sigma_E, L.R_E, L._XEinv[0] else: return None, None, None, None
def _generate_constraints(n=15, p=20, sigma=1): while True: y = np.random.standard_normal(n) * sigma beta = np.zeros(p) X = np.random.standard_normal((n,p)) + 0.3 * np.random.standard_normal(n)[:,None] X /= (X.std(0)[None,:] * np.sqrt(n)) y += np.dot(X, beta) * sigma lam_theor = 0.3 * choose_lambda(X, quantile=0.9) L = sqrt_lasso(y, X, lam_theor) L.fit(tol=1.e-12, min_its=150) con = L.active_constraints if con is not None and L.active.shape[0] >= 3: break con.covariance = np.identity(con.covariance.shape[0]) con.mean *= 0 return con, y, L
def test_estimate_sigma(n=200, p=400, s=10, sigma=3.): y = np.random.standard_normal(n) * sigma beta = np.zeros(p) beta[:s] = 8 * (2 * np.random.binomial(1, 0.5, size=(s,)) - 1) X = np.random.standard_normal((n,p)) + 0.3 * np.random.standard_normal(n)[:,None] X /= (X.std(0)[None,:] * np.sqrt(n)) y += np.dot(X, beta) * sigma lam_theor = choose_lambda(X, quantile=0.9) L = sqrt_lasso(y, X, lam_theor) L.fit(tol=1.e-12, min_its=150) P = [] if L.active.shape[0] > 0: return L.sigma_hat / sigma, L.sigma_E / sigma, L.df_E else: return (None,) * 3
def _generate_constraints(n=15, p=20, sigma=1): while True: y = np.random.standard_normal(n) * sigma beta = np.zeros(p) X = np.random.standard_normal( (n, p)) + 0.3 * np.random.standard_normal(n)[:, None] X /= (X.std(0)[None, :] * np.sqrt(n)) y += np.dot(X, beta) * sigma lam_theor = 0.3 * choose_lambda(X, quantile=0.9) L = sqrt_lasso(y, X, lam_theor) L.fit(tol=1.e-12, min_its=150) con = L.active_constraints if con is not None and L.active.shape[0] >= 3: break con.covariance = np.identity(con.covariance.shape[0]) con.mean *= 0 return con, y, L
def test_class(n=20, p=40, s=2): y = np.random.standard_normal(n) * 1.2 beta = np.zeros(p) beta[:s] = 5 X = np.random.standard_normal((n,p)) + 0.3 * np.random.standard_normal(n)[:,None] y += np.dot(X, beta) lam_theor = 0.7 * choose_lambda(X, quantile=0.9) L = sqrt_lasso(y,X,lam_theor) L.fit(tol=1.e-10, min_its=80) P = [] if L.active.shape[0] > 0: np.testing.assert_array_less( \ np.dot(L.constraints.linear_part, L.y), L.constraints.offset) nt.assert_true(L.constraints(y)) nt.assert_true(L.quasi_affine_constraints(y)) if set(range(s)).issubset(L.active): P = [p[1] for p in L.active_pvalues[s:]] else: P = [] return P