def report(niter=1, **kwargs):
#kwargs = {'s':3, 'n':300, 'p':20, 'signal':7, 'nview':4, 'test': 'global'}
kwargs = {'s': 3, 'n': 300, 'p': 20, 'signal': 7, 'nview': 1}
kwargs['bootstrap'] = False
intervals_report = reports.reports['test_multiple_queries']
CLT_runs = reports.collect_multiple_runs(intervals_report['test'],
intervals_report['columns'],
niter, reports.summarize_all,
**kwargs)
#fig = reports.pivot_plot(CLT_runs, color='b', label='CLT')
fig = reports.pivot_plot_2in1(CLT_runs, color='b', label='CLT')
kwargs['bootstrap'] = True
bootstrap_runs = reports.collect_multiple_runs(intervals_report['test'],
intervals_report['columns'],
niter,
reports.summarize_all,
**kwargs)
#fig = reports.pivot_plot(bootstrap_runs, color='g', label='Bootstrap', fig=fig)
fig = reports.pivot_plot_2in1(bootstrap_runs,
color='g',
label='Bootstrap',
fig=fig)
fig.savefig(
'multiple_queries.pdf') # will have both bootstrap and CLT on plot
def report_both(niter=10, **kwargs):
kwargs = {
's': 0,
'n': 500,
'p': 100,
'signal': 7,
'bootstrap': False,
'randomizer': 'gaussian'
}
intervals_report = reports.reports['test_intervals']
CLT_runs = reports.collect_multiple_runs(intervals_report['test'],
intervals_report['columns'],
niter, reports.summarize_all,
**kwargs)
#fig = reports.pivot_plot(CLT_runs, color='b', label='CLT')
fig = reports.pivot_plot_2in1(CLT_runs, color='b', label='CLT')
kwargs['bootstrap'] = True
bootstrap_runs = reports.collect_multiple_runs(intervals_report['test'],
intervals_report['columns'],
niter,
reports.summarize_all,
**kwargs)
#fig = reports.pivot_plot(bootstrap_runs, color='g', label='Bootstrap', fig=fig)
fig = reports.pivot_plot_2in1(bootstrap_runs,
color='g',
label='Bootstrap',
fig=fig)
fig.savefig(
'intervals_pivots.pdf') # will have both bootstrap and CLT on plot
def report(niter=50, **kwargs):
# these are all our null tests
fn_names = ['test_gaussian_pvals',
'test_logistic_pvals',
'test_data_carving_gaussian',
'test_data_carving_sqrt_lasso',
'test_data_carving_logistic',
'test_data_carving_poisson',
'test_data_carving_coxph'
]
dfs = []
for fn in fn_names:
fn = reports.reports[fn]
dfs.append(reports.collect_multiple_runs(fn['test'],
fn['columns'],
niter,
reports.summarize_all))
dfs = pd.concat(dfs)
fig = reports.pvalue_plot(dfs)
fig.savefig('algorithms_pvalues.pdf')
fig = reports.split_pvalue_plot(dfs)
fig.savefig('algorithms_split_pvalues.pdf')
def report(niter=50, **kwargs):
_report = goodness_of_fit_report = reports.reports['test_goodness_of_fit']
runs = reports.collect_multiple_runs(_report['test'], _report['columns'],
niter, reports.summarize_all,
**kwargs)
fig = reports.pvalue_plot(runs)
fig.savefig('sqrtlasso_goodness_of_fit.pdf')
def report(niter=50, **kwargs):
condition_report = reports.reports['test_condition']
runs = reports.collect_multiple_runs(condition_report['test'],
condition_report['columns'], niter,
reports.summarize_all, **kwargs)
fig = reports.pivot_plot_simple(runs)
fig.savefig('conditional_pivots.pdf')
def report(niter=50, **kwargs):
split_report = reports.reports['test_split']
CLT_runs = reports.collect_multiple_runs(split_report['test'],
split_report['columns'],
niter,
reports.summarize_all,
**kwargs)
kwargs['bootstrap'] = False
fig = reports.pivot_plot(CLT_runs, color='b', label='CLT')
kwargs['bootstrap'] = True
bootstrap_runs = reports.collect_multiple_runs(split_report['test'],
split_report['columns'],
niter,
reports.summarize_all,
**kwargs)
fig = reports.pivot_plot(bootstrap_runs, color='g', label='Bootstrap', fig=fig)
fig.savefig('split_pivots.pdf') # will have both bootstrap and CLT on plot
def report(niter=50, **kwargs):
fixedX_report = reports.reports['test_fixedX']
runs = reports.collect_multiple_runs(fixedX_report['test'],
fixedX_report['columns'], niter,
reports.summarize_all, **kwargs)
fig = reports.pvalue_plot(runs)
fig.savefig(
'fixedX_pivots.pdf') # will have both bootstrap and CLT on plot
def report(niter=50, **kwargs):
condition_report = reports.reports['test_marginalize']
runs = reports.collect_multiple_runs(condition_report['test'],
condition_report['columns'], niter,
reports.summarize_all, **kwargs)
fig = reports.pivot_plot_plus_naive(runs)
#fig = reports.pivot_plot_2in1(runs,color='b', label='marginalized subgradient')
fig.suptitle('Randomized Lasso marginalized subgradient')
fig.savefig('marginalized_subgrad_pivots.pdf')
def report(niter=200, **kwargs):
kwargs = {'s': 0, 'n': 200, 'p': 20, 'snr': 7, 'loss': 'gaussian', 'randomizer': 'gaussian'}
split_report = reports.reports['test_approximate_ci']
screened_results = reports.collect_multiple_runs(split_report['test'],
split_report['columns'],
niter,
reports.summarize_all,
**kwargs)
fig = reports.pivot_plot_plus_naive(screened_results)
fig.savefig('approx_pivots_threshold.pdf')
def report(niter=3, **kwargs):
kwargs = {'s': 0, 'n': 300, 'p': 20, 'signal': 7, 'split_frac': 0.8}
split_report = reports.reports['test_split_compare']
screened_results = reports.collect_multiple_runs(split_report['test'],
split_report['columns'],
niter,
reports.summarize_all,
**kwargs)
fig = reports.boot_clt_plot(screened_results, inactive=True, active=False)
fig.savefig('split_compare_pivots.pdf') # will have both bootstrap and CLT on plot
def report(niter=100, **kwargs):
kwargs = {'s': 0, 'n': 300, 'p': 10, 'signal': 7}
split_report = reports.reports['test_nonrandomized']
screened_results = reports.collect_multiple_runs(split_report['test'],
split_report['columns'],
niter,
reports.summarize_all,
**kwargs)
fig = reports.pivot_plot_simple(screened_results)
fig.savefig(
'nonrandomized_pivots.pdf') # will have both bootstrap and CLT on plot
def report(niter=50, **kwargs):
split_report = reports.reports['test_split_compare']
screened_results = reports.collect_multiple_runs(split_report['test'],
split_report['columns'],
niter,
reports.summarize_all,
**kwargs)
fig = reports.boot_clt_plot(screened_results,
color='b',
inactive=True,
active=False)
fig.savefig(
'split_compare_pivots.pdf') # will have both bootstrap and CLT on plot
def report(niter=50, **kwargs):
# these are all our null tests
fn_names = ['test_threshold_score']
dfs = []
for fn in fn_names:
fn = reports.reports[fn]
dfs.append(
reports.collect_multiple_runs(fn['test'], fn['columns'], niter,
reports.summarize_all))
dfs = pd.concat(dfs)
fig = reports.pvalue_plot(dfs, colors=['r', 'g'])
fig.savefig(
'threshold_pvalues.pdf') # will have both bootstrap and CLT on plot
def report(niter=100, design="random", **kwargs):
if design == "fixed":
X, _, _, _, _ = gaussian_instance(**kwargs)
kwargs.update({'X': X})
intervals_report = reports.reports['test_cv_corrected_nonrandomized_lasso']
screened_results = reports.collect_multiple_runs(
intervals_report['test'], intervals_report['columns'], niter,
reports.summarize_all, **kwargs)
screened_results.to_pickle("cv_corrected_nonrandomized_lasso.pkl")
results = pd.read_pickle("cv_corrected_nonrandomized_lasso.pkl")
fig = reports.pvalue_plot(results, label='CV corrected')
fig.suptitle("CV corrected norandomized Lasso pivots", fontsize=20)
fig.savefig('cv_corrected_nonrandomized_lasso_pivots.pdf')
def report(niter=50, **kwargs):
kwargs = {
's': 0,
'n': 600,
'p': 100,
'signal': 7,
'bootstrap': False,
'randomizer': 'gaussian',
'loss': 'gaussian',
'intervals': 'old'
}
intervals_report = reports.reports['test_intervals']
runs = reports.collect_multiple_runs(intervals_report['test'],
intervals_report['columns'], niter,
reports.summarize_all, **kwargs)
fig = reports.pivot_plot_plus_naive(runs)
fig.suptitle('Selective vs naive p-values after group Lasso')
fig.savefig('Group_lasso.pdf')
def report(niter=50, **kwargs):
# these are all our null tests
fn_names = ['test_parametric_covariance_small',
'test_multiple_queries_small',
'test_multiple_queries_individual_coeff_small']
dfs = []
for fn in fn_names:
fn = reports.reports[fn]
dfs.append(reports.collect_multiple_runs(fn['test'],
fn['columns'],
niter,
reports.summarize_all))
dfs = pd.concat(dfs)
fig = reports.pvalue_plot(dfs, colors=['r', 'g'])
fig.savefig('randomization_to_zero_pvalues.pdf') # will have both bootstrap and CLT on plot
def report(niter=50, design="random", **kwargs):
if design == "fixed":
X, _, _, _, _ = gaussian_instance(**kwargs)
kwargs.update({'X': X})
kwargs.update({'cross_validation': True, 'condition_on_CVR': False})
intervals_report = reports.reports['test_naive']
screened_results = reports.collect_multiple_runs(
intervals_report['test'], intervals_report['columns'], niter,
reports.summarize_all, **kwargs)
screened_results.to_pickle("naive.pkl")
results = pd.read_pickle("naive.pkl")
fig = reports.naive_pvalue_plot(results)
#fig = reports.pvalue_plot(results, label="Naive p-values")
fig.suptitle("Naive p-values", fontsize=20)
fig.savefig('naive_pvalues.pdf')
def report(niter=1, **kwargs):
condition_report = reports.reports['test_without_screening']
runs = reports.collect_multiple_runs(condition_report['test'],
condition_report['columns'], niter,
reports.summarize_all, **kwargs)
pkl_label = ''.join(["test_without_screening.pkl", "_", kwargs['loss'],"_",\
kwargs['randomizer'], ".pkl"])
pdf_label = ''.join(["test_without_screening.pkl", "_", kwargs['loss'], "_", \
kwargs['randomizer'], ".pdf"])
runs.to_pickle(pkl_label)
runs_read = pd.read_pickle(pkl_label)
fig = reports.pivot_plot_plus_naive(runs_read,
color='b',
label='no screening')
fig.suptitle('Testing without screening', fontsize=20)
fig.savefig(pdf_label)
def report(niter=10, **kwargs):
kwargs = {
's': 0,
'n': 300,
'p': 10,
'signal': 7,
'nviews': 3,
'intervals': 'old'
}
split_report = reports.reports['test_multiple_queries']
screened_results = reports.collect_multiple_runs(split_report['test'],
split_report['columns'],
niter,
reports.summarize_all,
**kwargs)
fig = reports.boot_clt_plot(screened_results, inactive=True, active=False)
fig.savefig(
'multiple_queries_CI.pdf') # will have both bootstrap and CLT on plot
def report(niter=50):
# these are all our null tests
fn_names = [
'test_overall_null_two_queries',
'test_one_inactive_coordinate_handcoded'
]
dfs = []
for fn in fn_names:
fn = reports.reports[fn]
dfs.append(
reports.collect_multiple_runs(fn['test'], fn['columns'], niter,
reports.summarize_all))
dfs = pd.concat(dfs)
fig = reports.pvalue_plot(dfs, colors=['r', 'g'])
fig = reports.naive_pvalue_plot(dfs, fig=fig, colors=['k', 'b'])
fig.savefig('Mest_pvalues.pdf') # will have both bootstrap and CLT on plot
def report(niter=50, **kwargs):
np.random.seed(500)
intervals_report = reports.reports['test_cv']
runs = reports.collect_multiple_runs(intervals_report['test'],
intervals_report['columns'], niter,
reports.summarize_all, **kwargs)
pkl_label = ''.join([
kwargs['loss'], "_",
str(kwargs['condition_on_CVR']), "_", "test_cv.pkl"
])
pdf_label = ''.join([
kwargs['loss'], "_",
str(kwargs['condition_on_CVR']), "_", "test_cv.pdf"
])
runs.to_pickle(pkl_label)
runs_read = pd.read_pickle(pkl_label)
fig = reports.pivot_plot_plus_naive(runs_read)
fig.suptitle("CV pivots", fontsize=20)
fig.savefig(pdf_label)
def report(niter=100, design="random", **kwargs):
if design == "fixed":
X, _, _, _, _ = gaussian_instance(**kwargs)
kwargs.update({'X': X})
intervals_report = reports.reports['test_lee_et_al']
screened_results = reports.collect_multiple_runs(
intervals_report['test'], intervals_report['columns'], niter,
reports.summarize_all, **kwargs)
screened_results.to_pickle("lee_et_al_pivots.pkl")
results = pd.read_pickle("lee_et_al_pivots.pkl")
#naive plus lee et al.
fig = reports.pivot_plot_plus_naive(results)
fig.suptitle("Lee et al. and naive p-values", fontsize=20)
fig.savefig('lee_et_al_pivots.pdf')
# naive only
fig1 = reports.naive_pvalue_plot(results)
fig1.suptitle("Naive p-values", fontsize=20)
fig1.savefig('naive_pvalues.pdf')
