def report(niter=1, **kwargs): #kwargs = {'s':3, 'n':300, 'p':20, 'signal':7, 'nview':4, 'test': 'global'} kwargs = {'s': 3, 'n': 300, 'p': 20, 'signal': 7, 'nview': 1} kwargs['bootstrap'] = False intervals_report = reports.reports['test_multiple_queries'] CLT_runs = reports.collect_multiple_runs(intervals_report['test'], intervals_report['columns'], niter, reports.summarize_all, **kwargs) #fig = reports.pivot_plot(CLT_runs, color='b', label='CLT') fig = reports.pivot_plot_2in1(CLT_runs, color='b', label='CLT') kwargs['bootstrap'] = True bootstrap_runs = reports.collect_multiple_runs(intervals_report['test'], intervals_report['columns'], niter, reports.summarize_all, **kwargs) #fig = reports.pivot_plot(bootstrap_runs, color='g', label='Bootstrap', fig=fig) fig = reports.pivot_plot_2in1(bootstrap_runs, color='g', label='Bootstrap', fig=fig) fig.savefig( 'multiple_queries.pdf') # will have both bootstrap and CLT on plot
def report_both(niter=10, **kwargs): kwargs = { 's': 0, 'n': 500, 'p': 100, 'signal': 7, 'bootstrap': False, 'randomizer': 'gaussian' } intervals_report = reports.reports['test_intervals'] CLT_runs = reports.collect_multiple_runs(intervals_report['test'], intervals_report['columns'], niter, reports.summarize_all, **kwargs) #fig = reports.pivot_plot(CLT_runs, color='b', label='CLT') fig = reports.pivot_plot_2in1(CLT_runs, color='b', label='CLT') kwargs['bootstrap'] = True bootstrap_runs = reports.collect_multiple_runs(intervals_report['test'], intervals_report['columns'], niter, reports.summarize_all, **kwargs) #fig = reports.pivot_plot(bootstrap_runs, color='g', label='Bootstrap', fig=fig) fig = reports.pivot_plot_2in1(bootstrap_runs, color='g', label='Bootstrap', fig=fig) fig.savefig( 'intervals_pivots.pdf') # will have both bootstrap and CLT on plot
def report(niter=50, **kwargs): # these are all our null tests fn_names = ['test_gaussian_pvals', 'test_logistic_pvals', 'test_data_carving_gaussian', 'test_data_carving_sqrt_lasso', 'test_data_carving_logistic', 'test_data_carving_poisson', 'test_data_carving_coxph' ] dfs = [] for fn in fn_names: fn = reports.reports[fn] dfs.append(reports.collect_multiple_runs(fn['test'], fn['columns'], niter, reports.summarize_all)) dfs = pd.concat(dfs) fig = reports.pvalue_plot(dfs) fig.savefig('algorithms_pvalues.pdf') fig = reports.split_pvalue_plot(dfs) fig.savefig('algorithms_split_pvalues.pdf')
def report(niter=50, **kwargs): _report = goodness_of_fit_report = reports.reports['test_goodness_of_fit'] runs = reports.collect_multiple_runs(_report['test'], _report['columns'], niter, reports.summarize_all, **kwargs) fig = reports.pvalue_plot(runs) fig.savefig('sqrtlasso_goodness_of_fit.pdf')
def report(niter=50, **kwargs): condition_report = reports.reports['test_condition'] runs = reports.collect_multiple_runs(condition_report['test'], condition_report['columns'], niter, reports.summarize_all, **kwargs) fig = reports.pivot_plot_simple(runs) fig.savefig('conditional_pivots.pdf')
def report(niter=50, **kwargs): split_report = reports.reports['test_split'] CLT_runs = reports.collect_multiple_runs(split_report['test'], split_report['columns'], niter, reports.summarize_all, **kwargs) kwargs['bootstrap'] = False fig = reports.pivot_plot(CLT_runs, color='b', label='CLT') kwargs['bootstrap'] = True bootstrap_runs = reports.collect_multiple_runs(split_report['test'], split_report['columns'], niter, reports.summarize_all, **kwargs) fig = reports.pivot_plot(bootstrap_runs, color='g', label='Bootstrap', fig=fig) fig.savefig('split_pivots.pdf') # will have both bootstrap and CLT on plot
def report(niter=50, **kwargs): fixedX_report = reports.reports['test_fixedX'] runs = reports.collect_multiple_runs(fixedX_report['test'], fixedX_report['columns'], niter, reports.summarize_all, **kwargs) fig = reports.pvalue_plot(runs) fig.savefig( 'fixedX_pivots.pdf') # will have both bootstrap and CLT on plot
def report(niter=50, **kwargs): condition_report = reports.reports['test_marginalize'] runs = reports.collect_multiple_runs(condition_report['test'], condition_report['columns'], niter, reports.summarize_all, **kwargs) fig = reports.pivot_plot_plus_naive(runs) #fig = reports.pivot_plot_2in1(runs,color='b', label='marginalized subgradient') fig.suptitle('Randomized Lasso marginalized subgradient') fig.savefig('marginalized_subgrad_pivots.pdf')
def report(niter=200, **kwargs): kwargs = {'s': 0, 'n': 200, 'p': 20, 'snr': 7, 'loss': 'gaussian', 'randomizer': 'gaussian'} split_report = reports.reports['test_approximate_ci'] screened_results = reports.collect_multiple_runs(split_report['test'], split_report['columns'], niter, reports.summarize_all, **kwargs) fig = reports.pivot_plot_plus_naive(screened_results) fig.savefig('approx_pivots_threshold.pdf')
def report(niter=3, **kwargs): kwargs = {'s': 0, 'n': 300, 'p': 20, 'signal': 7, 'split_frac': 0.8} split_report = reports.reports['test_split_compare'] screened_results = reports.collect_multiple_runs(split_report['test'], split_report['columns'], niter, reports.summarize_all, **kwargs) fig = reports.boot_clt_plot(screened_results, inactive=True, active=False) fig.savefig('split_compare_pivots.pdf') # will have both bootstrap and CLT on plot
def report(niter=100, **kwargs): kwargs = {'s': 0, 'n': 300, 'p': 10, 'signal': 7} split_report = reports.reports['test_nonrandomized'] screened_results = reports.collect_multiple_runs(split_report['test'], split_report['columns'], niter, reports.summarize_all, **kwargs) fig = reports.pivot_plot_simple(screened_results) fig.savefig( 'nonrandomized_pivots.pdf') # will have both bootstrap and CLT on plot
def report(niter=50, **kwargs): split_report = reports.reports['test_split_compare'] screened_results = reports.collect_multiple_runs(split_report['test'], split_report['columns'], niter, reports.summarize_all, **kwargs) fig = reports.boot_clt_plot(screened_results, color='b', inactive=True, active=False) fig.savefig( 'split_compare_pivots.pdf') # will have both bootstrap and CLT on plot
def report(niter=50, **kwargs): # these are all our null tests fn_names = ['test_threshold_score'] dfs = [] for fn in fn_names: fn = reports.reports[fn] dfs.append( reports.collect_multiple_runs(fn['test'], fn['columns'], niter, reports.summarize_all)) dfs = pd.concat(dfs) fig = reports.pvalue_plot(dfs, colors=['r', 'g']) fig.savefig( 'threshold_pvalues.pdf') # will have both bootstrap and CLT on plot
def report(niter=100, design="random", **kwargs): if design == "fixed": X, _, _, _, _ = gaussian_instance(**kwargs) kwargs.update({'X': X}) intervals_report = reports.reports['test_cv_corrected_nonrandomized_lasso'] screened_results = reports.collect_multiple_runs( intervals_report['test'], intervals_report['columns'], niter, reports.summarize_all, **kwargs) screened_results.to_pickle("cv_corrected_nonrandomized_lasso.pkl") results = pd.read_pickle("cv_corrected_nonrandomized_lasso.pkl") fig = reports.pvalue_plot(results, label='CV corrected') fig.suptitle("CV corrected norandomized Lasso pivots", fontsize=20) fig.savefig('cv_corrected_nonrandomized_lasso_pivots.pdf')
def report(niter=50, **kwargs): kwargs = { 's': 0, 'n': 600, 'p': 100, 'signal': 7, 'bootstrap': False, 'randomizer': 'gaussian', 'loss': 'gaussian', 'intervals': 'old' } intervals_report = reports.reports['test_intervals'] runs = reports.collect_multiple_runs(intervals_report['test'], intervals_report['columns'], niter, reports.summarize_all, **kwargs) fig = reports.pivot_plot_plus_naive(runs) fig.suptitle('Selective vs naive p-values after group Lasso') fig.savefig('Group_lasso.pdf')
def report(niter=50, **kwargs): # these are all our null tests fn_names = ['test_parametric_covariance_small', 'test_multiple_queries_small', 'test_multiple_queries_individual_coeff_small'] dfs = [] for fn in fn_names: fn = reports.reports[fn] dfs.append(reports.collect_multiple_runs(fn['test'], fn['columns'], niter, reports.summarize_all)) dfs = pd.concat(dfs) fig = reports.pvalue_plot(dfs, colors=['r', 'g']) fig.savefig('randomization_to_zero_pvalues.pdf') # will have both bootstrap and CLT on plot
def report(niter=50, design="random", **kwargs): if design == "fixed": X, _, _, _, _ = gaussian_instance(**kwargs) kwargs.update({'X': X}) kwargs.update({'cross_validation': True, 'condition_on_CVR': False}) intervals_report = reports.reports['test_naive'] screened_results = reports.collect_multiple_runs( intervals_report['test'], intervals_report['columns'], niter, reports.summarize_all, **kwargs) screened_results.to_pickle("naive.pkl") results = pd.read_pickle("naive.pkl") fig = reports.naive_pvalue_plot(results) #fig = reports.pvalue_plot(results, label="Naive p-values") fig.suptitle("Naive p-values", fontsize=20) fig.savefig('naive_pvalues.pdf')
def report(niter=1, **kwargs): condition_report = reports.reports['test_without_screening'] runs = reports.collect_multiple_runs(condition_report['test'], condition_report['columns'], niter, reports.summarize_all, **kwargs) pkl_label = ''.join(["test_without_screening.pkl", "_", kwargs['loss'],"_",\ kwargs['randomizer'], ".pkl"]) pdf_label = ''.join(["test_without_screening.pkl", "_", kwargs['loss'], "_", \ kwargs['randomizer'], ".pdf"]) runs.to_pickle(pkl_label) runs_read = pd.read_pickle(pkl_label) fig = reports.pivot_plot_plus_naive(runs_read, color='b', label='no screening') fig.suptitle('Testing without screening', fontsize=20) fig.savefig(pdf_label)
def report(niter=10, **kwargs): kwargs = { 's': 0, 'n': 300, 'p': 10, 'signal': 7, 'nviews': 3, 'intervals': 'old' } split_report = reports.reports['test_multiple_queries'] screened_results = reports.collect_multiple_runs(split_report['test'], split_report['columns'], niter, reports.summarize_all, **kwargs) fig = reports.boot_clt_plot(screened_results, inactive=True, active=False) fig.savefig( 'multiple_queries_CI.pdf') # will have both bootstrap and CLT on plot
def report(niter=50): # these are all our null tests fn_names = [ 'test_overall_null_two_queries', 'test_one_inactive_coordinate_handcoded' ] dfs = [] for fn in fn_names: fn = reports.reports[fn] dfs.append( reports.collect_multiple_runs(fn['test'], fn['columns'], niter, reports.summarize_all)) dfs = pd.concat(dfs) fig = reports.pvalue_plot(dfs, colors=['r', 'g']) fig = reports.naive_pvalue_plot(dfs, fig=fig, colors=['k', 'b']) fig.savefig('Mest_pvalues.pdf') # will have both bootstrap and CLT on plot
def report(niter=50, **kwargs): np.random.seed(500) intervals_report = reports.reports['test_cv'] runs = reports.collect_multiple_runs(intervals_report['test'], intervals_report['columns'], niter, reports.summarize_all, **kwargs) pkl_label = ''.join([ kwargs['loss'], "_", str(kwargs['condition_on_CVR']), "_", "test_cv.pkl" ]) pdf_label = ''.join([ kwargs['loss'], "_", str(kwargs['condition_on_CVR']), "_", "test_cv.pdf" ]) runs.to_pickle(pkl_label) runs_read = pd.read_pickle(pkl_label) fig = reports.pivot_plot_plus_naive(runs_read) fig.suptitle("CV pivots", fontsize=20) fig.savefig(pdf_label)
def report(niter=100, design="random", **kwargs): if design == "fixed": X, _, _, _, _ = gaussian_instance(**kwargs) kwargs.update({'X': X}) intervals_report = reports.reports['test_lee_et_al'] screened_results = reports.collect_multiple_runs( intervals_report['test'], intervals_report['columns'], niter, reports.summarize_all, **kwargs) screened_results.to_pickle("lee_et_al_pivots.pkl") results = pd.read_pickle("lee_et_al_pivots.pkl") #naive plus lee et al. fig = reports.pivot_plot_plus_naive(results) fig.suptitle("Lee et al. and naive p-values", fontsize=20) fig.savefig('lee_et_al_pivots.pdf') # naive only fig1 = reports.naive_pvalue_plot(results) fig1.suptitle("Naive p-values", fontsize=20) fig1.savefig('naive_pvalues.pdf')