if __name__ == "__main__": if load_run_data_flag: args, grid_results_dict = load_run_data(result_dir_to_load) else: grid_results_dict = run_simulations(args, local_mode) l2_grid = grid_results_dict['l2_grid'] gam_grid = grid_results_dict['gam_grid'] loss_avg = grid_results_dict['loss_avg'] loss_std = grid_results_dict['loss_std'] ci_factor = 1.96 / np.sqrt(args.n_reps) # 95% confidence interval factor max_deviate = 100. * np.max(loss_std * ci_factor / loss_avg) print('Max 95% CI relative to mean: ', max_deviate, '%') # fig, ax = plt.subplots(figsize=(7, 7)) with sns.axes_style("white"): yticklabels = np.around(l2_grid * 1e5, decimals=3) yticklabels = np.round(yticklabels).astype(int) xticklabels = np.around(gam_grid, decimals=3) ax = sns.heatmap(loss_avg, cmap="YlGnBu", xticklabels=xticklabels, yticklabels = yticklabels, annot=True, annot_kws={"size": 8}) ax.set_yticklabels(ax.get_yticklabels(), rotation=0) plt.xlabel(r'Guidance Discount Factor $\gamma$') plt.ylabel(r'$L_2$ Regularization Factor [1e-5]') if save_PDF: save_fig(args.run_name) else: plt.title('Loss avg. Max 95% CI relative to mean: {}%\n {}'.format(np.around(max_deviate, decimals=1), args.run_name)) plt.show() print('done')
def run_main_control(args, save_result=True, load_run_data_flag=False, result_dir_to_load='', save_PDF=False, plot=True, local_mode=False, init_ray=True): SetMdpArgs(args) if load_run_data_flag: args, info_dict = load_run_data(result_dir_to_load) else: info_dict = run_simulations(args, save_result, local_mode, init_ray=init_ray) planing_loss_avg = info_dict['planing_loss_avg'] planing_loss_std = info_dict['planing_loss_std'] alg_param_grid = info_dict['alg_param_grid'] if 'n_reps_finished' in info_dict.keys(): n_reps_finished = info_dict['n_reps_finished'] else: n_reps_finished = args.n_reps # end if # ----- Plot figures ---------------------------------------------# if plot or save_PDF: ax = plt.figure().gca() if args.train_sampling_def['type'] in { 'Generative', 'Generative_uniform', 'Generative_Stationary' }: data_size_per_traj = args.depth * args.n_episodes elif args.train_sampling_def['type'] == 'Trajectories': data_size_per_traj = args.depth * args.n_episodes elif args.train_sampling_def['type'] == 'sample_all_s_a': data_size_per_traj = args.nS * args.nA * args.n_episodes else: raise AssertionError # end if xscale = 1. legend_title = '' if args.param_grid_def['type'] == 'L2_factor': plt.xlabel(r'$L_2$ Regularization Factor [1e-2]') xscale = 1e2 elif args.param_grid_def['type'] == 'L1_factor': plt.xlabel(r'$L_1$ Regularization Factor ') elif args.param_grid_def['type'] == 'gamma_guidance': plt.xlabel(r'Guidance Discount Factor $\gamma$') else: raise AssertionError('Unrecognized args.grid_type') # end if ci_factor = 1.96 / np.sqrt( n_reps_finished) # 95% confidence interval factor config_grid_vals = get_grid(args.config_grid_def) for i_config, config_val in enumerate( config_grid_vals): # for of plots in the figure if args.config_grid_def['type'] == 'n_trajectories': if args.train_sampling_def['type'] in { 'Generative_uniform', 'Generative', 'Generative_Stationary' }: legend_title = 'Num. Samples' label_str = '{} '.format(config_val * data_size_per_traj) else: legend_title = 'Num. Trajectories' label_str = '{} '.format(config_val) elif args.config_grid_def[ 'type'] == 'states_actions_TV_dist_from_uniform': legend_title = 'Total-Variation from\n uniform [normalized]' label_str = '{} '.format(config_val) elif args.config_grid_def['type'] == 'chain_mix_time': legend_title = 'Mixing time' label_str = '{} '.format(config_val) elif args.config_grid_def['type'] == 'n_episodes': legend_title = 'Num. Episodes' label_str = '{} '.format(config_val) else: raise AssertionError # end if plt.errorbar(alg_param_grid * xscale, planing_loss_avg[i_config], yerr=planing_loss_std[i_config] * ci_factor, marker='.', label=label_str) if show_stars: # Mark the lowest point: i_best = np.argmin(planing_loss_avg[i_config]) plt.scatter(alg_param_grid[i_best] * xscale, planing_loss_avg[i_config][i_best], marker='*', s=400) # end if # for i_config plt.grid(True) plt.ylabel('Loss') plt.legend(title=legend_title, loc='best', fontsize=12) # loc='upper right' if y_lim: plt.ylim(y_lim) # plt.xlim([0.5,1]) # ax.set_yticks(np.arange(0., 9., step=1.)) # plt.figure(figsize=(5.8, 3.0)) # set up figure size if save_PDF: save_fig(args.run_name) else: # plt.title('Loss +- 95% CI \n ' + str(args.args)) plt.title(args.mdp_def['type'] + ' ' + args.run_name + ' \n ' + args.result_dir, fontsize=6) # end if save_PDF # end if pretty_print_args(args) if plot: plt.show() print('done') info_dict['result_dir'] = args.result_dir return info_dict
from utils.common_utils import set_default_plot_params, save_fig set_default_plot_params plt.figure() gammaDivGamma_e = np.linspace(0.1, 0.99, num=1000) # gammaEval = 0.99 # S = 10 # A = 2 # n_params = S * A # coeff = (gammaEval - gamma ) / (2 * gamma) coeff = 0.5 * (1 / gammaDivGamma_e - 1) # coeff /= n_params plt.plot(gammaDivGamma_e, (coeff)) plt.grid(True) plt.xlabel(r'$\gamma / \gamma_e$') plt.ylabel(r'$\lambda = \frac{\gamma_e - \gamma}{2\gamma}$') # plt.legend() # save_PDF = True # False \ True if save_PDF: save_fig('reg_coeff') else: plt.title('Activation Reg. Coeff.') plt.show() print('done')
ax.xaxis.set_major_locator(MaxNLocator(integer=True)) # plot loss ax = plt.figure().gca() ci_factor = 1.96 / np.sqrt(n_reps_finished) # 95% confidence interval factor for reg_type in reg_labels.keys(): plt.errorbar(xscale * hyper_grid_vals, best_loss_mean[reg_type], best_loss_std[reg_type] * ci_factor, label=reg_labels[reg_type]) plt.grid(True) plt.legend(fontsize=13) plt.xlabel(grid_label) plt.ylabel('Loss') if is_int_grid: ax.xaxis.set_major_locator(MaxNLocator(integer=True)) if save_PDF: save_fig(args.run_name, base_path=args.result_dir) # # # plot loss gain # ax = plt.figure().gca() # ci_factor = 1.96 / np.sqrt(n_reps_finished) # 95% confidence interval factor # for reg_type in reg_labels.keys(): # if reg_type != 'NoReg': # plt.errorbar(xscale * hyper_grid_vals, best_loss_mean['NoReg'] - best_loss_mean[reg_type], (best_loss_std[reg_type] + best_loss_std['NoReg'])* ci_factor, # label=reg_labels[reg_type]) # # TODO: more accurate std # plt.grid(True) # plt.legend() # plt.xlabel(grid_label) # plt.ylabel('Loss Gain') # if is_int_grid: # ax.xaxis.set_major_locator(MaxNLocator(integer=True))
def run_main_mrp(args, save_result=True, local_mode=local_mode, load_run_data_flag=False, result_dir_to_load='', save_PDF=False, plot=True, show_stars=True, y_lim=None, legend_loc='best'): SetMrpArgs(args) if load_run_data_flag: args, info_dict = load_run_data(result_dir_to_load) else: info_dict = run_simulations(args, save_result, local_mode) if 'loss_avg' in info_dict: loss_avg = info_dict['loss_avg'] loss_std = info_dict['loss_std'] else: loss_avg = info_dict['planing_loss_avg'] loss_std = info_dict['planing_loss_std'] if 'param_val_grid' in info_dict: param_val_grid = info_dict['param_val_grid'] else: param_val_grid = info_dict['alg_param_grid'] config_grid = info_dict['config_grid'] n_reps = args.n_reps # ----- Plot figures ---------------------------------------------# if plot or save_PDF: ax = plt.figure().gca() if args.train_sampling_def['type'] in { 'Generative', 'Generative_uniform', 'Generative_Stationary' }: data_size_per_traj = args.depth elif args.train_sampling_def['type'] == 'Trajectories': data_size_per_traj = args.depth elif args.train_sampling_def['type'] == 'sample_all_s': data_size_per_traj = args.nS else: raise AssertionError # end if xscale = 1. x_label = '' legend_title = '' grid_type = args.param_grid_def['type'] if grid_type == 'gamma_guidance': x_label = r'Guidance Discount Factor $\gamma$' elif grid_type == 'l2_TD': x_label = r'$L_2$ Regularization Factor [1e-3]' xscale = 1e3 elif grid_type == 'l2_proj': x_label = r'$L_2$ Regularization Factor - Projection Phase' elif grid_type == 'l2_fp': x_label = r'$L_2$ Regularization Factor - Fixed-Point Phase' elif grid_type == 'l2_TD': x_label = r'$L_2$ Regularization Factor' elif grid_type == 'l2_factor': x_label = r'$L_2$ Regularization Factor' elif grid_type == 'n_trajectories': if args.train_sampling_def['type'] in {'Generative', 'Generative_uniform'} \ or args.config_grid_def['type'] == 'states_TV_dist_from_uniform': xscale = args.depth x_label = 'Num. Samples' else: x_label = 'Num. Trajectories' # end if elif grid_type == 'states_TV_dist_from_uniform': x_label = 'Total-Variation from\n uniform [normalized]' # end if ci_factor = 1.96 / np.sqrt(n_reps) # 95% confidence interval factor # mixing_time_labels = ['Fast mixing', 'Moderate mixing', 'Slow mixing'] for i_config, config_val in enumerate( config_grid): # for of plots in the figure config_type = args.config_grid_def['type'] if config_type == 'p_left': args.mrp_def['p_left'] = config_val M = MRP(args) mixing_time = np.around(calc_typical_mixing_time(M.P), decimals=3) label_str = r'$\tau$={}'.format(mixing_time) print(label_str + r': $p_l=${} , $1/(1-|\lambda_2|)${}'.format( config_val, mixing_time)) elif config_type == 'forced_mix_time': label_str = '{}'.format(config_val) legend_title = 'Mixing-time' elif config_type == 'n_trajectories': if args.train_sampling_def['type'] in { 'Generative_uniform', 'Generative', 'Generative_Stationary' }: legend_title = 'Num. Samples' label_str = '{} '.format(config_val * data_size_per_traj) else: legend_title = 'Num. Trajectories' label_str = '{} '.format(config_val) elif config_type == 'states_TV_dist_from_uniform': legend_title = 'Total-Variation from\n uniform [normalized]' label_str = '{} '.format(config_val) elif config_type == 'None': legend_title = None label_str = '' elif config_type == 'RegMethod': legend_title = 'Regularizer' label_str = str(config_val[0]) + ': ' + str(config_val[1]) else: raise AssertionError plt.errorbar(xscale * param_val_grid, loss_avg[i_config], yerr=loss_std[i_config] * ci_factor, marker='.', label=label_str) if show_stars: # Mark the lowest point: i_best = np.argmin(loss_avg[i_config]) plt.scatter(xscale * param_val_grid[i_best], loss_avg[i_config][i_best], marker='*', s=400) print(label_str + ' Best x-coord: ' + str(xscale * param_val_grid[i_best])) # end if # for i_config plt.grid(True) plt.ylabel(r'Loss') plt.xlabel(x_label) if args.evaluation_loss_type == 'L2': plt.ylabel(r'$L_2$ Loss') if args.evaluation_loss_type == 'L2_uni_weight': plt.ylabel(r'Avg. $L_{2}$ Loss') elif args.evaluation_loss_type == 'rankings_kendalltau': plt.ylabel(r'Ranking Loss') if legend_title is not None: plt.legend(title=legend_title, loc=legend_loc, fontsize=12, title_fontsize=12) # loc='upper right' # plt.xlim([0.95,0.99]) if y_lim: plt.ylim(y_lim) # ax.set_yticks(np.arange(0., 9., step=1.)) if save_PDF: save_fig(args.run_name) else: # plt.title('Loss +- 95% CI \n ' + str(args.args)) plt.title(args.mdp_def['type'] + ' ' + args.run_name + ' \n ' + args.result_dir, fontsize=6) # end if save_PDF # end if pretty_print_args(args) if plot: plt.show() info_dict['result_dir'] = args.result_dir return info_dict