def plot_runtime(ex, fname, func_xvalues, xlabel, func_title=None): results = glo.ex_load_result(ex, fname) value_accessor = lambda job_results: job_results['time_secs'] vf_pval = np.vectorize(value_accessor) # results['test_results'] is a dictionary: # {'test_result': (dict from running perform_test(te) '...':..., } times = vf_pval(results['test_results']) repeats, _, n_methods = results['test_results'].shape time_avg = np.mean(times, axis=0) time_std = np.std(times, axis=0) xvalues = func_xvalues(results) #ns = np.array(results[xkey]) #te_proportion = 1.0 - results['tr_proportion'] #test_sizes = ns*te_proportion line_styles = exglo.func_plot_fmt_map() method_labels = exglo.get_func2label_map() func_names = [f.__name__ for f in results['method_job_funcs']] for i in range(n_methods): te_proportion = 1.0 - results['tr_proportion'] fmt = line_styles[func_names[i]] #plt.errorbar(ns*te_proportion, mean_rejs[:, i], std_pvals[:, i]) method_label = method_labels[func_names[i]] plt.errorbar(xvalues, time_avg[:, i], yerr=time_std[:, i], fmt=fmt, label=method_label) ylabel = 'Time (s)' plt.ylabel(ylabel) plt.xlabel(xlabel) plt.gca().set_yscale('log') plt.xlim([np.min(xvalues), np.max(xvalues)]) plt.xticks(xvalues, xvalues) plt.legend(loc='best') title = '%s. %d trials. ' % ( results['prob_label'], repeats) if func_title is None else func_title(results) plt.title(title) #plt.grid() return results
def plot_runtime(ex, fname, func_xvalues, xlabel, func_title=None): results = glo.ex_load_result(ex, fname) value_accessor = lambda job_results: job_results['time_secs'] vf_pval = np.vectorize(value_accessor) # results['test_results'] is a dictionary: # {'test_result': (dict from running perform_test(te) '...':..., } times = vf_pval(results['test_results']) repeats, _, n_methods = results['test_results'].shape time_avg = np.mean(times, axis=0) time_std = np.std(times, axis=0) xvalues = func_xvalues(results) #ns = np.array(results[xkey]) #te_proportion = 1.0 - results['tr_proportion'] #test_sizes = ns*te_proportion line_styles = exglo.func_plot_fmt_map() method_labels = exglo.get_func2label_map() func_names = [f.__name__ for f in results['method_job_funcs'] ] for i in range(n_methods): te_proportion = 1.0 - results['tr_proportion'] fmt = line_styles[func_names[i]] #plt.errorbar(ns*te_proportion, mean_rejs[:, i], std_pvals[:, i]) method_label = method_labels[func_names[i]] plt.errorbar(xvalues, time_avg[:, i], yerr=time_std[:,i], fmt=fmt, label=method_label) ylabel = 'Time (s)' plt.ylabel(ylabel) plt.xlabel(xlabel) plt.gca().set_yscale('log') plt.xlim([np.min(xvalues), np.max(xvalues)]) plt.xticks( xvalues, xvalues) plt.legend(loc='best') title = '%s. %d trials. '%( results['prob_label'], repeats ) if func_title is None else func_title(results) plt.title(title) #plt.grid() return results
def plot_prob_stat_above_thresh(ex, fname, h1_true, func_xvalues, xlabel, func_title=None): """ plot the empirical probability that the statistic is above the theshold. This can be interpreted as type-1 error (when H0 is true) or test power (when H1 is true). The plot is against the specified x-axis. - ex: experiment number - fname: file name of the aggregated result - h1_true: True if H1 is true - func_xvalues: function taking results dictionary and return the values to be used for the x-axis values. - xlabel: label of the x-axis. - func_title: a function: results dictionary -> title of the plot Return loaded results """ results = glo.ex_load_result(ex, fname) f_pval = lambda job_result: job_result['test_result']['h0_rejected'] #f_pval = lambda job_result: job_result['h0_rejected'] vf_pval = np.vectorize(f_pval) pvals = vf_pval(results['test_results']) repeats, _, n_methods = results['test_results'].shape mean_rejs = np.mean(pvals, axis=0) #std_pvals = np.std(pvals, axis=0) #std_pvals = np.sqrt(mean_rejs*(1.0-mean_rejs)) xvalues = func_xvalues(results) #ns = np.array(results[xkey]) #te_proportion = 1.0 - results['tr_proportion'] #test_sizes = ns*te_proportion line_styles = exglo.func_plot_fmt_map() method_labels = exglo.get_func2label_map() func_names = [f.__name__ for f in results['method_job_funcs']] for i in range(n_methods): te_proportion = 1.0 - results['tr_proportion'] fmt = line_styles[func_names[i]] #plt.errorbar(ns*te_proportion, mean_rejs[:, i], std_pvals[:, i]) method_label = method_labels[func_names[i]] plt.plot(xvalues, mean_rejs[:, i], fmt, label=method_label) ''' else: # h0 is true z = stats.norm.isf( (1-confidence)/2.0) for i in range(n_methods): phat = mean_rejs[:, i] conf_iv = z*(phat*(1-phat)/repeats)**0.5 #plt.errorbar(test_sizes, phat, conf_iv, fmt=line_styles[i], label=method_labels[i]) plt.plot(test_sizes, mean_rejs[:, i], line_styles[i], label=method_labels[i]) ''' ylabel = 'Test power' if h1_true else 'Type-I error' plt.ylabel(ylabel) plt.xlabel(xlabel) plt.xticks(np.hstack((xvalues))) alpha = results['alpha'] """ if not h1_true: # plot Wald interval if H0 is true # https://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval z = stats.norm.isf( (1-confidence)/2.0) gap = z*(alpha*(1-alpha)/repeats)**0.5 lb = alpha-gap ub = alpha+gap plt.plot(test_sizes, np.repeat(lb, len(test_sizes)), '--', linewidth=2, label='99%-Conf', color='k') plt.plot(test_sizes, np.repeat(ub, len(test_sizes)), '--', linewidth=2, color='k') plt.ylim([lb-0.005, ub+0.005]) """ plt.legend(loc='best') title = '%s. %d trials. $\\alpha$ = %.2g.' % ( results['prob_label'], repeats, alpha) if func_title is None else func_title(results) plt.title(title) #plt.grid() return results
def run_dataset(prob_label): """Run the experiment""" sample_source, n = get_sample_source(prob_label) # /////// submit jobs ////////// # create folder name string home = os.path.expanduser("~") foldername = os.path.join(home, "freqopttest_slurm", 'e%d' % ex) logger.info("Setting engine folder to %s" % foldername) # create parameter instance that is needed for any batch computation engine logger.info("Creating batch parameter instance") batch_parameters = BatchClusterParameters(foldername=foldername, job_name_base="e%d_" % ex, parameter_prefix="") # Use the following line if Slurm queue is not used. #engine = SerialComputationEngine() engine = SlurmComputationEngine(batch_parameters, do_clean_up=True) n_methods = len(method_job_funcs) # repetitions x #methods aggregators = np.empty((reps, n_methods), dtype=object) d = sample_source.dim() for r in range(reps): for mi, f in enumerate(method_job_funcs): # name used to save the result func_name = f.__name__ fname = '%s-%s-J%d_r%d_d%d_a%.3f_trp%.2f.p' \ %(prob_label, func_name, J, r, d, alpha, tr_proportion) if not is_rerun and glo.ex_file_exists(ex, prob_label, fname): logger.info('%s exists. Load and return.' % fname) test_result = glo.ex_load_result(ex, prob_label, fname) sra = SingleResultAggregator() sra.submit_result(SingleResult(test_result)) aggregators[r, mi] = sra else: # result not exists or rerun job = Ex5Job(SingleResultAggregator(), prob_label, r, n, f) agg = engine.submit_job(job) aggregators[r, mi] = agg # let the engine finish its business logger.info("Wait for all call in engine") engine.wait_for_all() # ////// collect the results /////////// logger.info("Collecting results") test_results = np.empty((reps, n_methods), dtype=object) for r in range(reps): for mi, f in enumerate(method_job_funcs): logger.info("Collecting result (%s, r=%d)" % (f.__name__, r)) # let the aggregator finalize things aggregators[r, mi].finalize() # aggregators[i].get_final_result() returns a SingleResult instance, # which we need to extract the actual result test_result = aggregators[r, mi].get_final_result().result test_results[r, mi] = test_result func_name = f.__name__ fname = '%s-%s-J%d_r%d_d%d_a%.3f_trp%.2f.p' \ %(prob_label, func_name, J, r, d, alpha, tr_proportion) glo.ex_save_result(ex, test_result, prob_label, fname) func_names = [f.__name__ for f in method_job_funcs] func2labels = exglobal.get_func2label_map() method_labels = [func2labels[f] for f in func_names if f in func2labels] # save results results = { 'results': test_results, 'n': n, 'data_fname': label2fname[prob_label], 'alpha': alpha, 'J': J, 'sample_source': sample_source, 'tr_proportion': tr_proportion, 'method_job_funcs': method_job_funcs, 'prob_label': prob_label, 'method_labels': method_labels } # class name fname = 'ex%d-%s-me%d_J%d_rs%d_nma%d_d%d_a%.3f_trp%.2f.p' \ %(ex, prob_label, n_methods, J, reps, n, d, alpha, tr_proportion) glo.ex_save_result(ex, results, fname) logger.info('Saved aggregated results to %s' % fname)
def run_dataset(prob_label): """Run the experiment""" sample_source, n = get_sample_source(prob_label) # /////// submit jobs ////////// # create folder name string home = os.path.expanduser("~") foldername = os.path.join(home, "freqopttest_slurm", 'e%d'%ex) logger.info("Setting engine folder to %s" % foldername) # create parameter instance that is needed for any batch computation engine logger.info("Creating batch parameter instance") batch_parameters = BatchClusterParameters( foldername=foldername, job_name_base="e%d_"%ex, parameter_prefix="") # Use the following line if Slurm queue is not used. #engine = SerialComputationEngine() engine = SlurmComputationEngine(batch_parameters, do_clean_up=True) n_methods = len(method_job_funcs) # repetitions x #methods aggregators = np.empty((reps, n_methods ), dtype=object) d = sample_source.dim() for r in range(reps): for mi, f in enumerate(method_job_funcs): # name used to save the result func_name = f.__name__ fname = '%s-%s-J%d_r%d_d%d_a%.3f_trp%.2f.p' \ %(prob_label, func_name, J, r, d, alpha, tr_proportion) if not is_rerun and glo.ex_file_exists(ex, prob_label, fname): logger.info('%s exists. Load and return.'%fname) test_result = glo.ex_load_result(ex, prob_label, fname) sra = SingleResultAggregator() sra.submit_result(SingleResult(test_result)) aggregators[r, mi] = sra else: # result not exists or rerun job = Ex5Job(SingleResultAggregator(), prob_label, r, n, f) agg = engine.submit_job(job) aggregators[r, mi] = agg # let the engine finish its business logger.info("Wait for all call in engine") engine.wait_for_all() # ////// collect the results /////////// logger.info("Collecting results") test_results = np.empty((reps, n_methods), dtype=object) for r in range(reps): for mi, f in enumerate(method_job_funcs): logger.info("Collecting result (%s, r=%d)" % (f.__name__, r )) # let the aggregator finalize things aggregators[r, mi].finalize() # aggregators[i].get_final_result() returns a SingleResult instance, # which we need to extract the actual result test_result = aggregators[r, mi].get_final_result().result test_results[r, mi] = test_result func_name = f.__name__ fname = '%s-%s-J%d_r%d_d%d_a%.3f_trp%.2f.p' \ %(prob_label, func_name, J, r, d, alpha, tr_proportion) glo.ex_save_result(ex, test_result, prob_label, fname) func_names = [f.__name__ for f in method_job_funcs] func2labels = exglobal.get_func2label_map() method_labels = [func2labels[f] for f in func_names if f in func2labels] # save results results = {'results': test_results, 'n': n, 'data_fname':label2fname[prob_label], 'alpha': alpha, 'J': J, 'sample_source': sample_source, 'tr_proportion': 0.5, 'method_job_funcs': method_job_funcs, 'prob_label': prob_label, 'method_labels': method_labels} # class name fname = 'ex%d-%s-me%d_J%d_rs%d_nma%d_d%d_a%.3f_trp%.2f.p' \ %(ex, prob_label, n_methods, J, reps, n, d, alpha, tr_proportion) glo.ex_save_result(ex, results, fname) logger.info('Saved aggregated results to %s'%fname)
# Use the following line if Slurm queue is not used. #engine = SerialComputationEngine() engine = SlurmComputationEngine(batch_parameters) n_methods = len(method_job_funcs) # repetitions x #methods aggregators = np.empty((reps, n_methods ), dtype=object) d = sample_source.dim() for r in range(reps): for mi, f in enumerate(method_job_funcs): # name used to save the result func_name = f.__name__ fname = '%s-%s-J%d_r%d_d%d_a%.3f_trp%.2f.p' \ %(prob_label, func_name, J, r, d, alpha, tr_proportion) if not is_rerun and glo.ex_file_exists(ex, prob_label, fname): logger.info('%s exists. Load and return.'%fname) test_result = glo.ex_load_result(ex, prob_label, fname) sra = SingleResultAggregator() if test_result is SingleResult: sra.submit_result(test_result) else: sra.submit_result(SingleResult(test_result)) aggregators[r, mi] = sra else: # result not exists or rerun job = Ex4Job(SingleResultAggregator(), prob_label, r, n, f) agg = engine.submit_job(job) aggregators[r, mi] = agg # let the engine finish its business
def plot_prob_stat_above_thresh(ex, fname, h1_true, func_xvalues, xlabel, func_title=None): """ plot the empirical probability that the statistic is above the theshold. This can be interpreted as type-1 error (when H0 is true) or test power (when H1 is true). The plot is against the specified x-axis. - ex: experiment number - fname: file name of the aggregated result - h1_true: True if H1 is true - func_xvalues: function taking results dictionary and return the values to be used for the x-axis values. - xlabel: label of the x-axis. - func_title: a function: results dictionary -> title of the plot Return loaded results """ results = glo.ex_load_result(ex, fname) f_pval = lambda job_result: job_result['test_result']['h0_rejected'] #f_pval = lambda job_result: job_result['h0_rejected'] vf_pval = np.vectorize(f_pval) pvals = vf_pval(results['test_results']) repeats, _, n_methods = results['test_results'].shape mean_rejs = np.mean(pvals, axis=0) #std_pvals = np.std(pvals, axis=0) #std_pvals = np.sqrt(mean_rejs*(1.0-mean_rejs)) xvalues = func_xvalues(results) #ns = np.array(results[xkey]) #te_proportion = 1.0 - results['tr_proportion'] #test_sizes = ns*te_proportion line_styles = exglo.func_plot_fmt_map() method_labels = exglo.get_func2label_map() func_names = [f.__name__ for f in results['method_job_funcs'] ] for i in range(n_methods): te_proportion = 1.0 - results['tr_proportion'] fmt = line_styles[func_names[i]] #plt.errorbar(ns*te_proportion, mean_rejs[:, i], std_pvals[:, i]) method_label = method_labels[func_names[i]] plt.plot(xvalues, mean_rejs[:, i], fmt, label=method_label) ''' else: # h0 is true z = stats.norm.isf( (1-confidence)/2.0) for i in range(n_methods): phat = mean_rejs[:, i] conf_iv = z*(phat*(1-phat)/repeats)**0.5 #plt.errorbar(test_sizes, phat, conf_iv, fmt=line_styles[i], label=method_labels[i]) plt.plot(test_sizes, mean_rejs[:, i], line_styles[i], label=method_labels[i]) ''' ylabel = 'Test power' if h1_true else 'Type-I error' plt.ylabel(ylabel) plt.xlabel(xlabel) plt.xticks( np.hstack((xvalues) )) alpha = results['alpha'] """ if not h1_true: # plot Wald interval if H0 is true # https://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval z = stats.norm.isf( (1-confidence)/2.0) gap = z*(alpha*(1-alpha)/repeats)**0.5 lb = alpha-gap ub = alpha+gap plt.plot(test_sizes, np.repeat(lb, len(test_sizes)), '--', linewidth=2, label='99%-Conf', color='k') plt.plot(test_sizes, np.repeat(ub, len(test_sizes)), '--', linewidth=2, color='k') plt.ylim([lb-0.005, ub+0.005]) """ plt.legend(loc='best') title = '%s. %d trials. $\\alpha$ = %.2g.'%( results['prob_label'], repeats, alpha) if func_title is None else func_title(results) plt.title(title) #plt.grid() return results