def plot_runtime(ex, fname, func_xvalues, xlabel, func_title=None):
results = glo.ex_load_result(ex, fname)
value_accessor = lambda job_results: job_results['time_secs']
vf_pval = np.vectorize(value_accessor)
# results['test_results'] is a dictionary:
# {'test_result': (dict from running perform_test(te) '...':..., }
times = vf_pval(results['test_results'])
repeats, _, n_methods = results['test_results'].shape
time_avg = np.mean(times, axis=0)
time_std = np.std(times, axis=0)
xvalues = func_xvalues(results)
#ns = np.array(results[xkey])
#te_proportion = 1.0 - results['tr_proportion']
#test_sizes = ns*te_proportion
line_styles = exglo.func_plot_fmt_map()
method_labels = exglo.get_func2label_map()
func_names = [f.__name__ for f in results['method_job_funcs']]
for i in range(n_methods):
te_proportion = 1.0 - results['tr_proportion']
fmt = line_styles[func_names[i]]
#plt.errorbar(ns*te_proportion, mean_rejs[:, i], std_pvals[:, i])
method_label = method_labels[func_names[i]]
plt.errorbar(xvalues,
time_avg[:, i],
yerr=time_std[:, i],
fmt=fmt,
label=method_label)
ylabel = 'Time (s)'
plt.ylabel(ylabel)
plt.xlabel(xlabel)
plt.gca().set_yscale('log')
plt.xlim([np.min(xvalues), np.max(xvalues)])
plt.xticks(xvalues, xvalues)
plt.legend(loc='best')
title = '%s. %d trials. ' % (
results['prob_label'],
repeats) if func_title is None else func_title(results)
plt.title(title)
#plt.grid()
return results
def plot_runtime(ex, fname, func_xvalues, xlabel, func_title=None):
results = glo.ex_load_result(ex, fname)
value_accessor = lambda job_results: job_results['time_secs']
vf_pval = np.vectorize(value_accessor)
# results['test_results'] is a dictionary:
# {'test_result': (dict from running perform_test(te) '...':..., }
times = vf_pval(results['test_results'])
repeats, _, n_methods = results['test_results'].shape
time_avg = np.mean(times, axis=0)
time_std = np.std(times, axis=0)
xvalues = func_xvalues(results)
#ns = np.array(results[xkey])
#te_proportion = 1.0 - results['tr_proportion']
#test_sizes = ns*te_proportion
line_styles = exglo.func_plot_fmt_map()
method_labels = exglo.get_func2label_map()
func_names = [f.__name__ for f in results['method_job_funcs'] ]
for i in range(n_methods):
te_proportion = 1.0 - results['tr_proportion']
fmt = line_styles[func_names[i]]
#plt.errorbar(ns*te_proportion, mean_rejs[:, i], std_pvals[:, i])
method_label = method_labels[func_names[i]]
plt.errorbar(xvalues, time_avg[:, i], yerr=time_std[:,i], fmt=fmt,
label=method_label)
ylabel = 'Time (s)'
plt.ylabel(ylabel)
plt.xlabel(xlabel)
plt.gca().set_yscale('log')
plt.xlim([np.min(xvalues), np.max(xvalues)])
plt.xticks( xvalues, xvalues)
plt.legend(loc='best')
title = '%s. %d trials. '%( results['prob_label'],
repeats ) if func_title is None else func_title(results)
plt.title(title)
#plt.grid()
return results
def plot_prob_stat_above_thresh(ex,
fname,
h1_true,
func_xvalues,
xlabel,
func_title=None):
"""
plot the empirical probability that the statistic is above the theshold.
This can be interpreted as type-1 error (when H0 is true) or test power
(when H1 is true). The plot is against the specified x-axis.
- ex: experiment number
- fname: file name of the aggregated result
- h1_true: True if H1 is true
- func_xvalues: function taking results dictionary and return the values
to be used for the x-axis values.
- xlabel: label of the x-axis.
- func_title: a function: results dictionary -> title of the plot
Return loaded results
"""
results = glo.ex_load_result(ex, fname)
f_pval = lambda job_result: job_result['test_result']['h0_rejected']
#f_pval = lambda job_result: job_result['h0_rejected']
vf_pval = np.vectorize(f_pval)
pvals = vf_pval(results['test_results'])
repeats, _, n_methods = results['test_results'].shape
mean_rejs = np.mean(pvals, axis=0)
#std_pvals = np.std(pvals, axis=0)
#std_pvals = np.sqrt(mean_rejs*(1.0-mean_rejs))
xvalues = func_xvalues(results)
#ns = np.array(results[xkey])
#te_proportion = 1.0 - results['tr_proportion']
#test_sizes = ns*te_proportion
line_styles = exglo.func_plot_fmt_map()
method_labels = exglo.get_func2label_map()
func_names = [f.__name__ for f in results['method_job_funcs']]
for i in range(n_methods):
te_proportion = 1.0 - results['tr_proportion']
fmt = line_styles[func_names[i]]
#plt.errorbar(ns*te_proportion, mean_rejs[:, i], std_pvals[:, i])
method_label = method_labels[func_names[i]]
plt.plot(xvalues, mean_rejs[:, i], fmt, label=method_label)
'''
else:
# h0 is true
z = stats.norm.isf( (1-confidence)/2.0)
for i in range(n_methods):
phat = mean_rejs[:, i]
conf_iv = z*(phat*(1-phat)/repeats)**0.5
#plt.errorbar(test_sizes, phat, conf_iv, fmt=line_styles[i], label=method_labels[i])
plt.plot(test_sizes, mean_rejs[:, i], line_styles[i], label=method_labels[i])
'''
ylabel = 'Test power' if h1_true else 'Type-I error'
plt.ylabel(ylabel)
plt.xlabel(xlabel)
plt.xticks(np.hstack((xvalues)))
alpha = results['alpha']
"""
if not h1_true:
# plot Wald interval if H0 is true
# https://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval
z = stats.norm.isf( (1-confidence)/2.0)
gap = z*(alpha*(1-alpha)/repeats)**0.5
lb = alpha-gap
ub = alpha+gap
plt.plot(test_sizes, np.repeat(lb, len(test_sizes)), '--', linewidth=2,
label='99%-Conf', color='k')
plt.plot(test_sizes, np.repeat(ub, len(test_sizes)), '--', linewidth=2, color='k')
plt.ylim([lb-0.005, ub+0.005])
"""
plt.legend(loc='best')
title = '%s. %d trials. $\\alpha$ = %.2g.' % (
results['prob_label'], repeats,
alpha) if func_title is None else func_title(results)
plt.title(title)
#plt.grid()
return results
def run_dataset(prob_label):
"""Run the experiment"""
sample_source, n = get_sample_source(prob_label)
# /////// submit jobs //////////
# create folder name string
home = os.path.expanduser("~")
foldername = os.path.join(home, "freqopttest_slurm", 'e%d' % ex)
logger.info("Setting engine folder to %s" % foldername)
# create parameter instance that is needed for any batch computation engine
logger.info("Creating batch parameter instance")
batch_parameters = BatchClusterParameters(foldername=foldername,
job_name_base="e%d_" % ex,
parameter_prefix="")
# Use the following line if Slurm queue is not used.
#engine = SerialComputationEngine()
engine = SlurmComputationEngine(batch_parameters, do_clean_up=True)
n_methods = len(method_job_funcs)
# repetitions x #methods
aggregators = np.empty((reps, n_methods), dtype=object)
d = sample_source.dim()
for r in range(reps):
for mi, f in enumerate(method_job_funcs):
# name used to save the result
func_name = f.__name__
fname = '%s-%s-J%d_r%d_d%d_a%.3f_trp%.2f.p' \
%(prob_label, func_name, J, r, d, alpha, tr_proportion)
if not is_rerun and glo.ex_file_exists(ex, prob_label, fname):
logger.info('%s exists. Load and return.' % fname)
test_result = glo.ex_load_result(ex, prob_label, fname)
sra = SingleResultAggregator()
sra.submit_result(SingleResult(test_result))
aggregators[r, mi] = sra
else:
# result not exists or rerun
job = Ex5Job(SingleResultAggregator(), prob_label, r, n, f)
agg = engine.submit_job(job)
aggregators[r, mi] = agg
# let the engine finish its business
logger.info("Wait for all call in engine")
engine.wait_for_all()
# ////// collect the results ///////////
logger.info("Collecting results")
test_results = np.empty((reps, n_methods), dtype=object)
for r in range(reps):
for mi, f in enumerate(method_job_funcs):
logger.info("Collecting result (%s, r=%d)" % (f.__name__, r))
# let the aggregator finalize things
aggregators[r, mi].finalize()
# aggregators[i].get_final_result() returns a SingleResult instance,
# which we need to extract the actual result
test_result = aggregators[r, mi].get_final_result().result
test_results[r, mi] = test_result
func_name = f.__name__
fname = '%s-%s-J%d_r%d_d%d_a%.3f_trp%.2f.p' \
%(prob_label, func_name, J, r, d, alpha, tr_proportion)
glo.ex_save_result(ex, test_result, prob_label, fname)
func_names = [f.__name__ for f in method_job_funcs]
func2labels = exglobal.get_func2label_map()
method_labels = [func2labels[f] for f in func_names if f in func2labels]
# save results
results = {
'results': test_results,
'n': n,
'data_fname': label2fname[prob_label],
'alpha': alpha,
'J': J,
'sample_source': sample_source,
'tr_proportion': tr_proportion,
'method_job_funcs': method_job_funcs,
'prob_label': prob_label,
'method_labels': method_labels
}
# class name
fname = 'ex%d-%s-me%d_J%d_rs%d_nma%d_d%d_a%.3f_trp%.2f.p' \
%(ex, prob_label, n_methods, J, reps, n, d, alpha, tr_proportion)
glo.ex_save_result(ex, results, fname)
logger.info('Saved aggregated results to %s' % fname)
def run_dataset(prob_label):
"""Run the experiment"""
sample_source, n = get_sample_source(prob_label)
# /////// submit jobs //////////
# create folder name string
home = os.path.expanduser("~")
foldername = os.path.join(home, "freqopttest_slurm", 'e%d'%ex)
logger.info("Setting engine folder to %s" % foldername)
# create parameter instance that is needed for any batch computation engine
logger.info("Creating batch parameter instance")
batch_parameters = BatchClusterParameters(
foldername=foldername, job_name_base="e%d_"%ex, parameter_prefix="")
# Use the following line if Slurm queue is not used.
#engine = SerialComputationEngine()
engine = SlurmComputationEngine(batch_parameters, do_clean_up=True)
n_methods = len(method_job_funcs)
# repetitions x #methods
aggregators = np.empty((reps, n_methods ), dtype=object)
d = sample_source.dim()
for r in range(reps):
for mi, f in enumerate(method_job_funcs):
# name used to save the result
func_name = f.__name__
fname = '%s-%s-J%d_r%d_d%d_a%.3f_trp%.2f.p' \
%(prob_label, func_name, J, r, d, alpha, tr_proportion)
if not is_rerun and glo.ex_file_exists(ex, prob_label, fname):
logger.info('%s exists. Load and return.'%fname)
test_result = glo.ex_load_result(ex, prob_label, fname)
sra = SingleResultAggregator()
sra.submit_result(SingleResult(test_result))
aggregators[r, mi] = sra
else:
# result not exists or rerun
job = Ex5Job(SingleResultAggregator(), prob_label, r, n, f)
agg = engine.submit_job(job)
aggregators[r, mi] = agg
# let the engine finish its business
logger.info("Wait for all call in engine")
engine.wait_for_all()
# ////// collect the results ///////////
logger.info("Collecting results")
test_results = np.empty((reps, n_methods), dtype=object)
for r in range(reps):
for mi, f in enumerate(method_job_funcs):
logger.info("Collecting result (%s, r=%d)" % (f.__name__, r ))
# let the aggregator finalize things
aggregators[r, mi].finalize()
# aggregators[i].get_final_result() returns a SingleResult instance,
# which we need to extract the actual result
test_result = aggregators[r, mi].get_final_result().result
test_results[r, mi] = test_result
func_name = f.__name__
fname = '%s-%s-J%d_r%d_d%d_a%.3f_trp%.2f.p' \
%(prob_label, func_name, J, r, d, alpha, tr_proportion)
glo.ex_save_result(ex, test_result, prob_label, fname)
func_names = [f.__name__ for f in method_job_funcs]
func2labels = exglobal.get_func2label_map()
method_labels = [func2labels[f] for f in func_names if f in func2labels]
# save results
results = {'results': test_results, 'n': n, 'data_fname':label2fname[prob_label],
'alpha': alpha, 'J': J, 'sample_source': sample_source,
'tr_proportion': 0.5, 'method_job_funcs': method_job_funcs,
'prob_label': prob_label, 'method_labels': method_labels}
# class name
fname = 'ex%d-%s-me%d_J%d_rs%d_nma%d_d%d_a%.3f_trp%.2f.p' \
%(ex, prob_label, n_methods, J, reps, n, d, alpha, tr_proportion)
glo.ex_save_result(ex, results, fname)
logger.info('Saved aggregated results to %s'%fname)
# Use the following line if Slurm queue is not used.
#engine = SerialComputationEngine()
engine = SlurmComputationEngine(batch_parameters)
n_methods = len(method_job_funcs)
# repetitions x #methods
aggregators = np.empty((reps, n_methods ), dtype=object)
d = sample_source.dim()
for r in range(reps):
for mi, f in enumerate(method_job_funcs):
# name used to save the result
func_name = f.__name__
fname = '%s-%s-J%d_r%d_d%d_a%.3f_trp%.2f.p' \
%(prob_label, func_name, J, r, d, alpha, tr_proportion)
if not is_rerun and glo.ex_file_exists(ex, prob_label, fname):
logger.info('%s exists. Load and return.'%fname)
test_result = glo.ex_load_result(ex, prob_label, fname)
sra = SingleResultAggregator()
if test_result is SingleResult:
sra.submit_result(test_result)
else:
sra.submit_result(SingleResult(test_result))
aggregators[r, mi] = sra
else:
# result not exists or rerun
job = Ex4Job(SingleResultAggregator(), prob_label, r, n, f)
agg = engine.submit_job(job)
aggregators[r, mi] = agg
# let the engine finish its business
def plot_prob_stat_above_thresh(ex, fname, h1_true, func_xvalues, xlabel,
func_title=None):
"""
plot the empirical probability that the statistic is above the theshold.
This can be interpreted as type-1 error (when H0 is true) or test power
(when H1 is true). The plot is against the specified x-axis.
- ex: experiment number
- fname: file name of the aggregated result
- h1_true: True if H1 is true
- func_xvalues: function taking results dictionary and return the values
to be used for the x-axis values.
- xlabel: label of the x-axis.
- func_title: a function: results dictionary -> title of the plot
Return loaded results
"""
results = glo.ex_load_result(ex, fname)
f_pval = lambda job_result: job_result['test_result']['h0_rejected']
#f_pval = lambda job_result: job_result['h0_rejected']
vf_pval = np.vectorize(f_pval)
pvals = vf_pval(results['test_results'])
repeats, _, n_methods = results['test_results'].shape
mean_rejs = np.mean(pvals, axis=0)
#std_pvals = np.std(pvals, axis=0)
#std_pvals = np.sqrt(mean_rejs*(1.0-mean_rejs))
xvalues = func_xvalues(results)
#ns = np.array(results[xkey])
#te_proportion = 1.0 - results['tr_proportion']
#test_sizes = ns*te_proportion
line_styles = exglo.func_plot_fmt_map()
method_labels = exglo.get_func2label_map()
func_names = [f.__name__ for f in results['method_job_funcs'] ]
for i in range(n_methods):
te_proportion = 1.0 - results['tr_proportion']
fmt = line_styles[func_names[i]]
#plt.errorbar(ns*te_proportion, mean_rejs[:, i], std_pvals[:, i])
method_label = method_labels[func_names[i]]
plt.plot(xvalues, mean_rejs[:, i], fmt, label=method_label)
'''
else:
# h0 is true
z = stats.norm.isf( (1-confidence)/2.0)
for i in range(n_methods):
phat = mean_rejs[:, i]
conf_iv = z*(phat*(1-phat)/repeats)**0.5
#plt.errorbar(test_sizes, phat, conf_iv, fmt=line_styles[i], label=method_labels[i])
plt.plot(test_sizes, mean_rejs[:, i], line_styles[i], label=method_labels[i])
'''
ylabel = 'Test power' if h1_true else 'Type-I error'
plt.ylabel(ylabel)
plt.xlabel(xlabel)
plt.xticks( np.hstack((xvalues) ))
alpha = results['alpha']
"""
if not h1_true:
# plot Wald interval if H0 is true
# https://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval
z = stats.norm.isf( (1-confidence)/2.0)
gap = z*(alpha*(1-alpha)/repeats)**0.5
lb = alpha-gap
ub = alpha+gap
plt.plot(test_sizes, np.repeat(lb, len(test_sizes)), '--', linewidth=2,
label='99%-Conf', color='k')
plt.plot(test_sizes, np.repeat(ub, len(test_sizes)), '--', linewidth=2, color='k')
plt.ylim([lb-0.005, ub+0.005])
"""
plt.legend(loc='best')
title = '%s. %d trials. $\\alpha$ = %.2g.'%( results['prob_label'],
repeats, alpha) if func_title is None else func_title(results)
plt.title(title)
#plt.grid()
return results