def test_covtest():
n, p = 30, 50
X = np.random.standard_normal((n,p)) + np.random.standard_normal(n)[:,None]
X /= X.std(0)[None,:]
Y = np.random.standard_normal(n) * 1.5
for exact, covariance in itertools.product([True, False],
[None, np.identity(n)]):
con, pval, idx, sign = covtest(X, Y, sigma=1.5, exact=exact,
covariance=covariance)
for covariance in [None, np.identity(n)]:
con, pval, idx, sign = selected_covtest(X, Y, sigma=1.5,
covariance=covariance)
con, pval, idx, sign = selected_covtest(X, Y)
return pval
def test_tilting(nsim=100, ndraw=50000, burnin=10000):
P = []
covered0 = 0
coveredA = 0
screen = 0
for i in range(nsim):
X, Y, beta, active, sigma = instance(n=20, p=30)
Y0 = np.random.standard_normal(X.shape[0]) * sigma
# null pvalues and intervals
cone, pvalue, idx, sign = selected_covtest(X, Y0, sigma=sigma)
eta = X[:,idx] * sign
p1, _, _, fam = gibbs_test(cone, Y0, eta,
ndraw=ndraw,
burnin=burnin,
alternative='twosided',
sigma_known=True,
tilt=eta,
UMPU=False)
observed_value = (Y0 * eta).sum()
lower_lim, upper_lim = fam.equal_tailed_interval(observed_value)
lower_lim_final = np.dot(eta, np.dot(cone.covariance, eta)) * lower_lim
upper_lim_final = np.dot(eta, np.dot(cone.covariance, eta)) * upper_lim
covered0 += (lower_lim_final < 0) * (upper_lim_final > 0)
print(covered0 / (i + 1.), 'coverage0')
# compare to no tilting
p2 = gibbs_test(cone, Y0, X[:,idx] * sign,
ndraw=ndraw,
burnin=burnin,
alternative='twosided',
sigma_known=True,
tilt=None,
UMPU=False)[0]
print(p2, 'huh')
P.append((p1, p2))
Pa = np.array(P)
# p1 and p2 should be very close, so have high correlation
print(np.corrcoef(Pa.T)[0,1], 'correlation')
# they should also look uniform -- mean should be about 0.5, sd about 0.29
print(np.mean(Pa, 0), 'mean of nulls')
print(np.std(Pa, 0), 'sd of nulls')
# alternative intervals
mu = 3 * X[:,0] * sigma
YA = np.random.standard_normal(X.shape[0]) * sigma + mu
cone, pvalue, idx, sign = selected_covtest(X, YA, sigma=sigma)
_, _, _, fam = gibbs_test(cone, YA, X[:,idx] * sign,
ndraw=ndraw,
burnin=burnin,
alternative='greater',
sigma_known=True,
tilt=eta)
if idx == 0:
screen += 1
eta = X[:,0] * sign
observed_value = (YA * eta).sum()
target = (eta * mu).sum()
lower_lim, upper_lim = fam.equal_tailed_interval(observed_value)
lower_lim_final = np.dot(eta, np.dot(cone.covariance, eta)) * lower_lim
upper_lim_final = np.dot(eta, np.dot(cone.covariance, eta)) * upper_lim
print(lower_lim_final, upper_lim_final, target)
coveredA += (lower_lim_final < target) * (upper_lim_final > target)
print(coveredA / (screen * 1.), 'coverageA')
print(screen / (i + 1.), 'screening')
def test_tilting(nsim=100):
P = []
covered0 = 0
coveredA = 0
screen = 0
for i in range(nsim):
X, Y, beta, active, sigma = instance(n=20, p=30)
Y0 = np.random.standard_normal(X.shape[0]) * sigma
# null pvalues and intervals
cone, pvalue, idx, sign = selected_covtest(X, Y0, sigma=sigma)
eta = X[:,idx] * sign
p1, _, _, fam = gibbs_test(cone, Y0, eta,
ndraw=50000,
burnin=10000,
alternative='twosided',
sigma_known=True,
tilt=eta,
UMPU=False)
observed_value = (Y0 * eta).sum()
lower_lim, upper_lim = fam.equal_tailed_interval(observed_value)
lower_lim_final = np.dot(eta, np.dot(cone.covariance, eta)) * lower_lim
upper_lim_final = np.dot(eta, np.dot(cone.covariance, eta)) * upper_lim
covered0 += (lower_lim_final < 0) * (upper_lim_final > 0)
print covered0 / (i + 1.), 'coverage0'
# compare to no tilting
p2 = gibbs_test(cone, Y0, X[:,idx] * sign,
ndraw=50000,
burnin=10000,
alternative='twosided',
sigma_known=True,
tilt=None,
UMPU=False)[0]
print p2, 'huh'
P.append((p1, p2))
Pa = np.array(P)
# p1 and p2 should be very close, so have high correlation
print np.corrcoef(Pa.T)[0,1], 'correlation'
# they should also look uniform -- mean should be about 0.5, sd about 0.29
print np.mean(Pa, 0), 'mean of nulls'
print np.std(Pa, 0), 'sd of nulls'
# alternative intervals
mu = 3 * X[:,0] * sigma
YA = np.random.standard_normal(X.shape[0]) * sigma + mu
cone, pvalue, idx, sign = selected_covtest(X, YA, sigma=sigma)
_, _, _, fam = gibbs_test(cone, YA, X[:,idx] * sign,
ndraw=15000,
burnin=10000,
alternative='greater',
sigma_known=True,
tilt=eta)
if idx == 0:
screen += 1
eta = X[:,0] * sign
observed_value = (YA * eta).sum()
target = (eta * mu).sum()
lower_lim, upper_lim = fam.equal_tailed_interval(observed_value)
lower_lim_final = np.dot(eta, np.dot(cone.covariance, eta)) * lower_lim
upper_lim_final = np.dot(eta, np.dot(cone.covariance, eta)) * upper_lim
print lower_lim_final, upper_lim_final, target
coveredA += (lower_lim_final < target) * (upper_lim_final > target)
print coveredA / (screen * 1.), 'coverageA'
print screen / (i + 1.), 'screening'
plt.figure()
plt.scatter(Pa[:,0], Pa[:,1])
try:
import statsmodels.api as sm
plt.figure()
G = np.linspace(0, 1, 101)
plt.plot(G, sm.distributions.ECDF(Pa[:,0])(G))
plt.plot(G, sm.distributions.ECDF(Pa[:,1])(G))
except ImportError: # no statsmodels
pass
