if __name__ == "__main__":
if load_run_data_flag:
args, grid_results_dict = load_run_data(result_dir_to_load)
else:
grid_results_dict = run_simulations(args, local_mode)
l2_grid = grid_results_dict['l2_grid']
gam_grid = grid_results_dict['gam_grid']
loss_avg = grid_results_dict['loss_avg']
loss_std = grid_results_dict['loss_std']
ci_factor = 1.96 / np.sqrt(args.n_reps) # 95% confidence interval factor
max_deviate = 100. * np.max(loss_std * ci_factor / loss_avg)
print('Max 95% CI relative to mean: ', max_deviate, '%')
# fig, ax = plt.subplots(figsize=(7, 7))
with sns.axes_style("white"):
yticklabels = np.around(l2_grid * 1e5, decimals=3)
yticklabels = np.round(yticklabels).astype(int)
xticklabels = np.around(gam_grid, decimals=3)
ax = sns.heatmap(loss_avg, cmap="YlGnBu", xticklabels=xticklabels, yticklabels = yticklabels, annot=True, annot_kws={"size": 8})
ax.set_yticklabels(ax.get_yticklabels(), rotation=0)
plt.xlabel(r'Guidance Discount Factor $\gamma$')
plt.ylabel(r'$L_2$ Regularization Factor [1e-5]')
if save_PDF:
save_fig(args.run_name)
else:
plt.title('Loss avg. Max 95% CI relative to mean: {}%\n {}'.format(np.around(max_deviate, decimals=1), args.run_name))
plt.show()
print('done')
def run_main_control(args,
save_result=True,
load_run_data_flag=False,
result_dir_to_load='',
save_PDF=False,
plot=True,
local_mode=False,
init_ray=True):
SetMdpArgs(args)
if load_run_data_flag:
args, info_dict = load_run_data(result_dir_to_load)
else:
info_dict = run_simulations(args,
save_result,
local_mode,
init_ray=init_ray)
planing_loss_avg = info_dict['planing_loss_avg']
planing_loss_std = info_dict['planing_loss_std']
alg_param_grid = info_dict['alg_param_grid']
if 'n_reps_finished' in info_dict.keys():
n_reps_finished = info_dict['n_reps_finished']
else:
n_reps_finished = args.n_reps
# end if
# ----- Plot figures ---------------------------------------------#
if plot or save_PDF:
ax = plt.figure().gca()
if args.train_sampling_def['type'] in {
'Generative', 'Generative_uniform', 'Generative_Stationary'
}:
data_size_per_traj = args.depth * args.n_episodes
elif args.train_sampling_def['type'] == 'Trajectories':
data_size_per_traj = args.depth * args.n_episodes
elif args.train_sampling_def['type'] == 'sample_all_s_a':
data_size_per_traj = args.nS * args.nA * args.n_episodes
else:
raise AssertionError
# end if
xscale = 1.
legend_title = ''
if args.param_grid_def['type'] == 'L2_factor':
plt.xlabel(r'$L_2$ Regularization Factor [1e-2]')
xscale = 1e2
elif args.param_grid_def['type'] == 'L1_factor':
plt.xlabel(r'$L_1$ Regularization Factor ')
elif args.param_grid_def['type'] == 'gamma_guidance':
plt.xlabel(r'Guidance Discount Factor $\gamma$')
else:
raise AssertionError('Unrecognized args.grid_type')
# end if
ci_factor = 1.96 / np.sqrt(
n_reps_finished) # 95% confidence interval factor
config_grid_vals = get_grid(args.config_grid_def)
for i_config, config_val in enumerate(
config_grid_vals): # for of plots in the figure
if args.config_grid_def['type'] == 'n_trajectories':
if args.train_sampling_def['type'] in {
'Generative_uniform', 'Generative',
'Generative_Stationary'
}:
legend_title = 'Num. Samples'
label_str = '{} '.format(config_val * data_size_per_traj)
else:
legend_title = 'Num. Trajectories'
label_str = '{} '.format(config_val)
elif args.config_grid_def[
'type'] == 'states_actions_TV_dist_from_uniform':
legend_title = 'Total-Variation from\n uniform [normalized]'
label_str = '{} '.format(config_val)
elif args.config_grid_def['type'] == 'chain_mix_time':
legend_title = 'Mixing time'
label_str = '{} '.format(config_val)
elif args.config_grid_def['type'] == 'n_episodes':
legend_title = 'Num. Episodes'
label_str = '{} '.format(config_val)
else:
raise AssertionError
# end if
plt.errorbar(alg_param_grid * xscale,
planing_loss_avg[i_config],
yerr=planing_loss_std[i_config] * ci_factor,
marker='.',
label=label_str)
if show_stars:
# Mark the lowest point:
i_best = np.argmin(planing_loss_avg[i_config])
plt.scatter(alg_param_grid[i_best] * xscale,
planing_loss_avg[i_config][i_best],
marker='*',
s=400)
# end if
# for i_config
plt.grid(True)
plt.ylabel('Loss')
plt.legend(title=legend_title, loc='best',
fontsize=12) # loc='upper right'
if y_lim:
plt.ylim(y_lim)
# plt.xlim([0.5,1])
# ax.set_yticks(np.arange(0., 9., step=1.))
# plt.figure(figsize=(5.8, 3.0)) # set up figure size
if save_PDF:
save_fig(args.run_name)
else:
# plt.title('Loss +- 95% CI \n ' + str(args.args))
plt.title(args.mdp_def['type'] + ' ' + args.run_name + ' \n ' +
args.result_dir,
fontsize=6)
# end if save_PDF
# end if
pretty_print_args(args)
if plot:
plt.show()
print('done')
info_dict['result_dir'] = args.result_dir
return info_dict
from utils.common_utils import set_default_plot_params, save_fig
set_default_plot_params
plt.figure()
gammaDivGamma_e = np.linspace(0.1, 0.99, num=1000)
# gammaEval = 0.99
# S = 10
# A = 2
# n_params = S * A
# coeff = (gammaEval - gamma ) / (2 * gamma)
coeff = 0.5 * (1 / gammaDivGamma_e - 1)
# coeff /= n_params
plt.plot(gammaDivGamma_e, (coeff))
plt.grid(True)
plt.xlabel(r'$\gamma / \gamma_e$')
plt.ylabel(r'$\lambda = \frac{\gamma_e - \gamma}{2\gamma}$')
# plt.legend()
#
save_PDF = True # False \ True
if save_PDF:
save_fig('reg_coeff')
else:
plt.title('Activation Reg. Coeff.')
plt.show()
print('done')
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
# plot loss
ax = plt.figure().gca()
ci_factor = 1.96 / np.sqrt(n_reps_finished) # 95% confidence interval factor
for reg_type in reg_labels.keys():
plt.errorbar(xscale * hyper_grid_vals, best_loss_mean[reg_type], best_loss_std[reg_type] * ci_factor, label=reg_labels[reg_type])
plt.grid(True)
plt.legend(fontsize=13)
plt.xlabel(grid_label)
plt.ylabel('Loss')
if is_int_grid:
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
if save_PDF:
save_fig(args.run_name, base_path=args.result_dir)
#
# # plot loss gain
# ax = plt.figure().gca()
# ci_factor = 1.96 / np.sqrt(n_reps_finished) # 95% confidence interval factor
# for reg_type in reg_labels.keys():
# if reg_type != 'NoReg':
# plt.errorbar(xscale * hyper_grid_vals, best_loss_mean['NoReg'] - best_loss_mean[reg_type], (best_loss_std[reg_type] + best_loss_std['NoReg'])* ci_factor,
# label=reg_labels[reg_type])
# # TODO: more accurate std
# plt.grid(True)
# plt.legend()
# plt.xlabel(grid_label)
# plt.ylabel('Loss Gain')
# if is_int_grid:
# ax.xaxis.set_major_locator(MaxNLocator(integer=True))
def run_main_mrp(args,
save_result=True,
local_mode=local_mode,
load_run_data_flag=False,
result_dir_to_load='',
save_PDF=False,
plot=True,
show_stars=True,
y_lim=None,
legend_loc='best'):
SetMrpArgs(args)
if load_run_data_flag:
args, info_dict = load_run_data(result_dir_to_load)
else:
info_dict = run_simulations(args, save_result, local_mode)
if 'loss_avg' in info_dict:
loss_avg = info_dict['loss_avg']
loss_std = info_dict['loss_std']
else:
loss_avg = info_dict['planing_loss_avg']
loss_std = info_dict['planing_loss_std']
if 'param_val_grid' in info_dict:
param_val_grid = info_dict['param_val_grid']
else:
param_val_grid = info_dict['alg_param_grid']
config_grid = info_dict['config_grid']
n_reps = args.n_reps
# ----- Plot figures ---------------------------------------------#
if plot or save_PDF:
ax = plt.figure().gca()
if args.train_sampling_def['type'] in {
'Generative', 'Generative_uniform', 'Generative_Stationary'
}:
data_size_per_traj = args.depth
elif args.train_sampling_def['type'] == 'Trajectories':
data_size_per_traj = args.depth
elif args.train_sampling_def['type'] == 'sample_all_s':
data_size_per_traj = args.nS
else:
raise AssertionError
# end if
xscale = 1.
x_label = ''
legend_title = ''
grid_type = args.param_grid_def['type']
if grid_type == 'gamma_guidance':
x_label = r'Guidance Discount Factor $\gamma$'
elif grid_type == 'l2_TD':
x_label = r'$L_2$ Regularization Factor [1e-3]'
xscale = 1e3
elif grid_type == 'l2_proj':
x_label = r'$L_2$ Regularization Factor - Projection Phase'
elif grid_type == 'l2_fp':
x_label = r'$L_2$ Regularization Factor - Fixed-Point Phase'
elif grid_type == 'l2_TD':
x_label = r'$L_2$ Regularization Factor'
elif grid_type == 'l2_factor':
x_label = r'$L_2$ Regularization Factor'
elif grid_type == 'n_trajectories':
if args.train_sampling_def['type'] in {'Generative', 'Generative_uniform'} \
or args.config_grid_def['type'] == 'states_TV_dist_from_uniform':
xscale = args.depth
x_label = 'Num. Samples'
else:
x_label = 'Num. Trajectories'
# end if
elif grid_type == 'states_TV_dist_from_uniform':
x_label = 'Total-Variation from\n uniform [normalized]'
# end if
ci_factor = 1.96 / np.sqrt(n_reps) # 95% confidence interval factor
# mixing_time_labels = ['Fast mixing', 'Moderate mixing', 'Slow mixing']
for i_config, config_val in enumerate(
config_grid): # for of plots in the figure
config_type = args.config_grid_def['type']
if config_type == 'p_left':
args.mrp_def['p_left'] = config_val
M = MRP(args)
mixing_time = np.around(calc_typical_mixing_time(M.P),
decimals=3)
label_str = r'$\tau$={}'.format(mixing_time)
print(label_str + r': $p_l=${} , $1/(1-|\lambda_2|)${}'.format(
config_val, mixing_time))
elif config_type == 'forced_mix_time':
label_str = '{}'.format(config_val)
legend_title = 'Mixing-time'
elif config_type == 'n_trajectories':
if args.train_sampling_def['type'] in {
'Generative_uniform', 'Generative',
'Generative_Stationary'
}:
legend_title = 'Num. Samples'
label_str = '{} '.format(config_val * data_size_per_traj)
else:
legend_title = 'Num. Trajectories'
label_str = '{} '.format(config_val)
elif config_type == 'states_TV_dist_from_uniform':
legend_title = 'Total-Variation from\n uniform [normalized]'
label_str = '{} '.format(config_val)
elif config_type == 'None':
legend_title = None
label_str = ''
elif config_type == 'RegMethod':
legend_title = 'Regularizer'
label_str = str(config_val[0]) + ': ' + str(config_val[1])
else:
raise AssertionError
plt.errorbar(xscale * param_val_grid,
loss_avg[i_config],
yerr=loss_std[i_config] * ci_factor,
marker='.',
label=label_str)
if show_stars:
# Mark the lowest point:
i_best = np.argmin(loss_avg[i_config])
plt.scatter(xscale * param_val_grid[i_best],
loss_avg[i_config][i_best],
marker='*',
s=400)
print(label_str + ' Best x-coord: ' +
str(xscale * param_val_grid[i_best]))
# end if
# for i_config
plt.grid(True)
plt.ylabel(r'Loss')
plt.xlabel(x_label)
if args.evaluation_loss_type == 'L2':
plt.ylabel(r'$L_2$ Loss')
if args.evaluation_loss_type == 'L2_uni_weight':
plt.ylabel(r'Avg. $L_{2}$ Loss')
elif args.evaluation_loss_type == 'rankings_kendalltau':
plt.ylabel(r'Ranking Loss')
if legend_title is not None:
plt.legend(title=legend_title,
loc=legend_loc,
fontsize=12,
title_fontsize=12) # loc='upper right'
# plt.xlim([0.95,0.99])
if y_lim:
plt.ylim(y_lim)
# ax.set_yticks(np.arange(0., 9., step=1.))
if save_PDF:
save_fig(args.run_name)
else:
# plt.title('Loss +- 95% CI \n ' + str(args.args))
plt.title(args.mdp_def['type'] + ' ' + args.run_name + ' \n ' +
args.result_dir,
fontsize=6)
# end if save_PDF
# end if
pretty_print_args(args)
if plot:
plt.show()
info_dict['result_dir'] = args.result_dir
return info_dict