def find_best_perf(alg, exp, auc_or_final, sp):
exp_attrs = EXP_ATTRS[exp](exp)
fp_list, _, tp_list, fop_list, res_path = make_params(alg, exp)
best_params = {}
best_perf, best_fp, best_sp, best_tp, best_fop = np.inf, np.inf, np.inf, np.inf, np.inf
for fop in fop_list:
for tp in tp_list:
current_params = make_current_params(alg, sp, tp, fop)
load_name = os.path.join(
res_path,
create_name_for_save_load(current_params,
excluded_params=['alpha']) +
f'_mean_{auc_or_final}_over_alpha.npy')
current_perf = load_and_replace_large_nan_inf(
load_name,
large=exp_attrs.learning_starting_point,
replace_with=exp_attrs.over_limit_replacement)
min_perf = min(current_perf)
if min_perf < best_perf:
best_perf = min_perf
best_perf_idx = int(np.nanargmin(current_perf))
best_fp = fp_list[best_perf_idx]
best_params = current_params
best_params['alpha'] = best_fp
return best_params
def load_all_performances(alg, exp, auc_or_final, sp, exp_attrs):
fp_list, sp_list, tp_list, fop_list, res_path = make_params(alg, exp)
all_performance = np.zeros((len(fp_list), len(tp_list), len(fop_list)))
for i, fop in enumerate(fop_list):
for j, tp in enumerate(tp_list):
current_params = make_current_params(alg, sp, tp, fop)
load_file_name = os.path.join(
res_path,
create_name_for_save_load(current_params,
excluded_params=['alpha']) +
f'_mean_{auc_or_final}_over_alpha.npy')
if RERUN:
load_file_name_rerun = load_file_name.replace(
'.npy', f"{RERUN_POSTFIX}.npy")
if os.path.exists(load_file_name):
load_file_name = load_file_name_rerun
performance = np.load(load_file_name)
performance = replace_large_nan_inf(
performance,
large=exp_attrs.learning_starting_point,
replace_with=exp_attrs.over_limit_waterfall)
all_performance[:, j, i] = performance
return all_performance
def save_perf_over_alpha(alg, exp, auc_or_final, sp, rerun=False):
fp_list, sp_list, tp_list, fop_list, _ = make_params(alg, exp)
res_path = make_res_path(alg, exp)
mean_over_alpha, stderr_over_alpha = np.zeros(len(fp_list)), np.zeros(
len(fp_list))
best_fp, best_tp, best_fop = load_best_rerun_params(
alg, exp, auc_or_final, sp) if rerun else (0, 0, 0)
for tp in tp_list:
for fop in fop_list:
current_params = make_current_params(alg, sp, tp, fop)
for i, fp in enumerate(fp_list):
current_params['alpha'] = fp
load_name = os.path.join(
res_path, create_name_for_save_load(current_params))
perf = np.load(f"{load_name}_mean_stderr_{auc_or_final}.npy")
if rerun and fp == best_fp and tp == best_tp and fop == best_fop:
perf = np.load(
f"{load_name}_mean_stderr_{auc_or_final}_rerun.npy")
mean_over_alpha[i], stderr_over_alpha[i] = perf[0], perf[1]
save_name = os.path.join(
res_path,
create_name_for_save_load(current_params,
excluded_params=['alpha']))
postfix = ''
if rerun and tp == best_tp and fop == best_fop:
postfix = '_rerun'
np.save(f"{save_name}_mean_{auc_or_final}_over_alpha{postfix}",
mean_over_alpha)
np.save(f"{save_name}_stderr_{auc_or_final}_over_alpha{postfix}",
stderr_over_alpha)
def plot_all_sensitivities_per_alg():
for exp in EXPS:
exp_attrs = EXP_ATTRS[exp](exp)
for auc_or_final in AUC_AND_FINAL:
for sp in LMBDA_AND_ZETA:
for alg in SELECTED_ALGS:
save_dir = os.path.join('pdf_plots', 'AllThirds', exp,
f'Lmbda{sp}_{auc_or_final}')
fig, ax = plt.subplots()
fp_list, sp_list, tp_list, fop_list, _ = make_params(
alg, exp)
for tp in tp_list:
for fop in fop_list:
current_params = make_current_params(
alg, sp, tp, fop)
alphas = get_alphas(alg, exp)
performance, stderr = load_performance_over_alpha(
alg, exp, current_params, auc_or_final,
exp_attrs)
plot_sensitivity(ax, alg, exp, alphas, tp,
performance, stderr, exp_attrs)
if not os.path.exists(save_dir):
os.makedirs(save_dir, exist_ok=True)
fig.savefig(os.path.join(save_dir,
f"sensitivity_{alg}_{exp}.pdf"),
format='pdf',
dpi=1000,
bbox_inches='tight')
plt.show()
print(exp, alg, auc_or_final, sp)
def save_best_perf_in_json(alg, exp, best_params, auc_or_final, sp):
fp_list, _, tp_list, fop_list, res_path = make_params(alg, exp)
exp_path = res_path.replace('Results', 'Experiments')
json_exp = os.path.join(exp_path, f"{alg}.json")
with open(json_exp, 'r') as f:
json_exp = json.load(f)
json_exp['meta_parameters'] = best_params
save_name = os.path.join(res_path, f"{auc_or_final}_{sp}.json")
with open(save_name, 'wt') as f:
json.dump(json_exp, f, indent=4)