def save_perf_over_alpha(alg, exp, auc_or_final, sp, rerun=False):
fp_list, sp_list, tp_list, fop_list, _ = make_params(alg, exp)
res_path = make_res_path(alg, exp)
mean_over_alpha, stderr_over_alpha = np.zeros(len(fp_list)), np.zeros(
len(fp_list))
best_fp, best_tp, best_fop = load_best_rerun_params(
alg, exp, auc_or_final, sp) if rerun else (0, 0, 0)
for tp in tp_list:
for fop in fop_list:
current_params = make_current_params(alg, sp, tp, fop)
for i, fp in enumerate(fp_list):
current_params['alpha'] = fp
load_name = os.path.join(
res_path, create_name_for_save_load(current_params))
perf = np.load(f"{load_name}_mean_stderr_{auc_or_final}.npy")
if rerun and fp == best_fp and tp == best_tp and fop == best_fop:
perf = np.load(
f"{load_name}_mean_stderr_{auc_or_final}_rerun.npy")
mean_over_alpha[i], stderr_over_alpha[i] = perf[0], perf[1]
save_name = os.path.join(
res_path,
create_name_for_save_load(current_params,
excluded_params=['alpha']))
postfix = ''
if rerun and tp == best_tp and fop == best_fop:
postfix = '_rerun'
np.save(f"{save_name}_mean_{auc_or_final}_over_alpha{postfix}",
mean_over_alpha)
np.save(f"{save_name}_stderr_{auc_or_final}_over_alpha{postfix}",
stderr_over_alpha)
def find_best_perf(alg, exp, auc_or_final, sp):
exp_attrs = EXP_ATTRS[exp](exp)
fp_list, _, tp_list, fop_list, res_path = make_params(alg, exp)
best_params = {}
best_perf, best_fp, best_sp, best_tp, best_fop = np.inf, np.inf, np.inf, np.inf, np.inf
for fop in fop_list:
for tp in tp_list:
current_params = make_current_params(alg, sp, tp, fop)
load_name = os.path.join(
res_path,
create_name_for_save_load(current_params,
excluded_params=['alpha']) +
f'_mean_{auc_or_final}_over_alpha.npy')
current_perf = load_and_replace_large_nan_inf(
load_name,
large=exp_attrs.learning_starting_point,
replace_with=exp_attrs.over_limit_replacement)
min_perf = min(current_perf)
if min_perf < best_perf:
best_perf = min_perf
best_perf_idx = int(np.nanargmin(current_perf))
best_fp = fp_list[best_perf_idx]
best_params = current_params
best_params['alpha'] = best_fp
return best_params
def load_all_performances(alg, exp, auc_or_final, sp, exp_attrs):
fp_list, sp_list, tp_list, fop_list, res_path = make_params(alg, exp)
all_performance = np.zeros((len(fp_list), len(tp_list), len(fop_list)))
for i, fop in enumerate(fop_list):
for j, tp in enumerate(tp_list):
current_params = make_current_params(alg, sp, tp, fop)
load_file_name = os.path.join(
res_path,
create_name_for_save_load(current_params,
excluded_params=['alpha']) +
f'_mean_{auc_or_final}_over_alpha.npy')
if RERUN:
load_file_name_rerun = load_file_name.replace(
'.npy', f"{RERUN_POSTFIX}.npy")
if os.path.exists(load_file_name):
load_file_name = load_file_name_rerun
performance = np.load(load_file_name)
performance = replace_large_nan_inf(
performance,
large=exp_attrs.learning_starting_point,
replace_with=exp_attrs.over_limit_waterfall)
all_performance[:, j, i] = performance
return all_performance
def load_data(alg, exp, best_params, postfix=''):
res_path = os.path.join(os.getcwd(), 'Results', exp, alg)
generic_name = create_name_for_save_load(best_params)
load_file_name = os.path.join(res_path, f"{generic_name}_RMSVE_mean_over_runs{postfix}.npy")
mean_lc = np.load(load_file_name)
load_file_name = os.path.join(res_path, f"{generic_name}_RMSVE_stderr_over_runs{postfix}.npy")
stderr_lc = np.load(load_file_name)
return mean_lc, stderr_lc
def get_ls_rmsve(alg, exp, sp):
res_path = os.path.join(os.getcwd(), 'Results', exp, alg)
params = {'alpha': 0.01, 'lmbda': sp}
if alg == 'LSETD':
params['beta'] = 0.9
generic_name = create_name_for_save_load(params)
load_file_name = os.path.join(res_path,
f"{generic_name}_RMSVE_mean_over_runs.npy")
return np.load(load_file_name)
def load_performance_over_alpha(alg, exp, params, auc_or_final, exp_attrs):
res_path = make_res_path(alg, exp)
load_file_name = os.path.join(
res_path,
create_name_for_save_load(params, excluded_params=['alpha']) +
f"_mean_{auc_or_final}_over_alpha.npy")
performance_over_alpha = np.load(load_file_name)
performance_over_alpha = replace_large_nan_inf(
performance_over_alpha,
large=exp_attrs.learning_starting_point,
replace_with=exp_attrs.over_limit_replacement)
stderr_load_file_name = os.path.join(
res_path,
create_name_for_save_load(params, excluded_params=['alpha']) +
f'_stderr_{auc_or_final}_over_alpha.npy')
std_err_of_best_perf_over_alpha = np.load(stderr_load_file_name)
std_err_of_best_perf_over_alpha = replace_large_nan_inf(
std_err_of_best_perf_over_alpha,
large=exp_attrs.learning_starting_point,
replace_with=0.0)
return performance_over_alpha, std_err_of_best_perf_over_alpha
def load_data(alg, exp, best_params, postfix=''):
if exp in [
'FirstChain', 'FirstFourRoom', '1HVFourRoom', 'NewChain',
'NewFourRoom', 'NewHVFourRoom'
]:
evalu = 'RMSVE'
else:
evalu = 'steps'
res_path = os.path.join(os.getcwd(), 'Results', exp, alg)
generic_name = create_name_for_save_load(best_params)
load_file_name = os.path.join(
res_path, f"{generic_name}_{evalu}_mean_over_runs{postfix}.npy")
mean_lc = np.load(load_file_name)
print(mean_lc)
load_file_name = os.path.join(
res_path, f"{generic_name}_{evalu}_stderr_over_runs{postfix}.npy")
stderr_lc = np.load(load_file_name)
return mean_lc, stderr_lc
def rename_best_old_result(res_path, params_dict, file_name):
name_to_save = create_name_for_save_load(param_dict=params_dict)
path_and_name = os.path.join(res_path, name_to_save)
file_name = path_and_name + file_name
os.rename(file_name + '.npy', file_name + '_old.npy')