def make_regret_T_plots(res_files): for (T,N), res_file in res_files.items(): if T in plt_axes: pass else: _, median_ax = plt.subplots() _, mean_ax = plt.subplots() _, succ_ax = plt.subplots() plt_axes[T] = [median_ax, mean_ax, succ_ax] regret_results = util.read_from_file(res_file, verbose=False) Ls = sorted(regret_results.keys()) mean_regrets = [] median_regrets = [] std_dev_regrets = [] q25_regrets = [] q75_regrets = [] prob_successes = [] std_successes = [] for L in Ls: all_L_regrets = list(itertools.chain.from_iterable([regret_results[L][model] \ for model in regret_results[L].keys()])) mean_regrets += [np.mean(all_L_regrets)] median_regrets += [np.median(all_L_regrets)] std_dev_regrets += [np.std(all_L_regrets)] q25_regrets += [np.quantile(all_L_regrets, 0.25)] q75_regrets += [np.quantile(all_L_regrets, 0.75)] prob_success, std_success = get_success(all_L_regrets) prob_successes += [prob_success] std_successes += [std_success] med_bot, med_mid, med_top = q25_regrets, median_regrets, q75_regrets # Quantiles mean_bot, mean_mid, mean_top = np.subtract(mean_regrets, std_dev_regrets), \ mean_regrets, \ np.add(mean_regrets, std_dev_regrets) # Standard Deviation succ_bot, succ_mid, succ_top = np.subtract(prob_successes, std_successes), \ prob_successes, \ np.add(prob_successes, std_successes) # Success for (bot, mid, top, type, ax) in ((med_bot, med_mid, med_top, 'Median Regret', plt_axes[T][0]),\ (mean_bot, mean_mid, mean_top, 'Mean Regret', plt_axes[T][1]),\ (succ_bot, succ_mid, succ_top, '% Success', plt_axes[T][2])): #plt.figure() for plot_type in plot_info: if plot_type in name: plot_params = plot_info[plot_type] ax.plot(Ls, mid, c=plot_params[0], label=plot_params[1]) ax.fill_between(Ls, bot, top, facecolor=plot_params[0], alpha=0.2) ax.set_ylim(0, 1) ax.legend() ax.set_xlabel('L') ax.set_ylabel(type) ax.set_title('Evaluated on T=%s Interactions on N=%s Mechanisms' % (T, N))
def add_baseline_to_ax(fname, type, mean_ax, median_ax): plot_params = plot_info[type] steps = util.read_from_file(fname) mean = [np.mean(steps)]*10 std = [np.std(steps)]*10 median = [np.median(steps)]*10 q25 = [np.quantile(steps, 0.25)]*10 q75 = [np.quantile(steps, 0.75)]*10 Ls = list(range(0,101,11)) mean_ax.plot(Ls, mean, c=plot_params[0], label=plot_params[1]) mean_ax.fill_between(Ls, np.subtract(mean, std), np.add(mean,std), facecolor=plot_params[0], alpha=0.2) median_ax.plot(Ls, median, c=plot_params[0], label=plot_params[1]) median_ax.fill_between(Ls, q25, q75, facecolor=plot_params[0], alpha=0.2)
def __correct_box(arm_file, eng_file, out_file): lines = [] lines1 = util.read_from_file(arm_file); lines1 = ('\n'.join(lines1)).split(); lines2 = util.read_from_file(eng_file); content = '' if lines1 and lines2 and len(lines1) is len(lines2): for i in range (0,len(lines1)): if not (len(lines1[i]) == (2,1)): try: lines2[i] = lines2[i][lines2[i].index(' '):len(lines2[i])] content = content + lines1[i] + lines2[i] except ValueError: print(bcolors.FAIL + ' Incorrect box file coordinates ... ' + bcolors.ENDC) return else: print(bcolors.FAIL + ' Armenian file content is incorrect ... ' + bcolors.ENDC) return util.write_in_file(out_file, content) print (bcolors.OKGREEN + 'Done... \n' + 'Created ' + out_file + ' file ... ' + bcolors.ENDC) else: raise NotFound('Sizes of the ' + arm_file + ' and ' + eng_file + ' files are not equal ... or files are empty')
def __generate_and_correct_box_files(size_file, font_file): __copy_font_files(PATH_FONTS_DIR) fonts = util.read_from_file(font_file) sizes = util.read_json_from_file(size_file) if not (fonts is None or sizes is None): for size in sizes: try: size_lpi = size['lpi'] size_cpi = size['cpi'] except KeyError, e: print(bcolors.WARNING + 'Warning: size file content ... ' + bcolors.ENDC) return for font in fonts: __generate_and_correct_box_file(size_lpi, size_cpi, font)
def get_bb_dataset(bb_fname, n_bbs, mech_types, max_mech, urdf_num): # Create a dataset of busyboxes. if bb_fname == '' or bb_fname is None: print('Creating Busyboxes.') mech_classes = [] for mech_type in mech_types: if mech_type == 'slider': mech_classes.append(Slider) if mech_type == 'door': mech_classes.append(Door) bb_dataset = [] for _ in range(n_bbs): # TODO: i think there is a bug here... bb = BusyBox.generate_random_busybox(max_mech=max_mech, mech_types=mech_classes, urdf_tag=urdf_num) mechanism_params = bb._mechanisms[0].get_mechanism_tuple() image_data, gripper = setup_env(bb, False, False) bb_dataset.append([util.Result(None, mechanism_params, None, None, None, None, image_data, None)]) print('BusyBoxes created.') else: # Load in a file with predetermined BusyBoxes. bb_dataset = read_from_file(bb_fname)[:n_bbs] return bb_dataset
def add_points_to_saved_ax(explr_p, sample_points={}): fname = 'mean_fig.pickle' fig, axes, all_param_data, all_angular_params, all_linear_params = util.read_from_file( fname) policy_type = 'Prismatic' curr_ax_i = 0 for angular_param in all_angular_params: if not angular_param.varied: continue for linear_param in all_linear_params: if not linear_param.varied: continue # TODO: only works for up to 2 other_params (will have to figure out new # visualization past that) plt.suptitle(policy_type + ' mean fn:' + angular_param.param_name + \ ' vs ' + linear_param.param_name) # bin the other param values all_other_params = [ PlotData(param_name, param_data.varied, param_data.bounds) for (param_name, param_data) in all_param_data.items() if param_name != angular_param.param_name and param_name != linear_param.param_name and param_data.varied ] subplot_inds_and_vals = {} for other_param in all_other_params: subplot_inds_and_vals[other_param.param_name] = \ list(enumerate(np.linspace(*other_param.range, N_BINS))) if len(subplot_inds_and_vals) == 0: n_rows = 1 n_cols = 1 else: keys = list(subplot_inds_and_vals.keys()) if len(subplot_inds_and_vals) == 1: n_rows = 1 n_cols = len(subplot_inds_and_vals[keys[0]]) elif len(subplot_inds_and_vals) == 2: n_rows = len(subplot_inds_and_vals[keys[0]]) n_cols = len(subplot_inds_and_vals[keys[1]]) for single_subplot_inds_and_vals in itertools.product( *subplot_inds_and_vals.values()): if len(single_subplot_inds_and_vals) == 0: subplot_num = 1 else: row_col = [o[0] + 1 for o in single_subplot_inds_and_vals] if len(row_col) == 1: subplot_num = 1 * row_col[0] else: subplot_num = reduce(lambda x, y: n_cols * (x - 1) + y, row_col) # make polar subplot of mean function axes[curr_ax_i].set_title('\n'.join([ str(all_other_params[other_param_i].param_name) + ' = ' + str("%.2f" % other_val) for other_param_i, ( subplot_i, other_val) in enumerate(single_subplot_inds_and_vals) ]), fontsize=10) # only add points to subplot that are close to this subplot "bin" plot_points = [] pt_colors = sample_points[policy_type] if subplot_inds_and_vals == {}: plot_points = [(get_plot_point(x, angular_param.param_name, linear_param.param_name, all_param_data, policy_type), c) for (x, c) in pt_colors] else: for pt, color in pt_colors: keep_point = True for other_param_name, (subplot_dim, subplot_val) in \ zip(subplot_inds_and_vals, single_subplot_inds_and_vals): all_subplot_vals = np.array([ inds_and_vals[1] for inds_and_vals in subplot_inds_and_vals[other_param_name] ]) point_param_val = pt[x_inds[other_param_name]] closest_index = (np.abs(all_subplot_vals - point_param_val)).argmin() if all_subplot_vals[closest_index] == subplot_val: keep_point = keep_point and True else: keep_point = keep_point and False if keep_point: plot_points.append( (get_plot_point(pt, angular_param.param_name, linear_param.param_name, all_param_data, policy_type), color)) add_points(axes[curr_ax_i], plot_points) curr_ax_i += 1 #plt.show() fig.savefig('voo_testing/omega_%f.png' % explr_p)
def make_regret_plots(types, res_path, nonCPP): _, median_ax = plt.subplots() _, mean_ax = plt.subplots() # add nonCPP baselines to plots if nonCPP: for file in nonCPP: if 'random' in file: type = 'noncpp_random' elif 'gpucb' in file: type = 'noncpp_gpucb' else: print("could not detect data type of nonCPP baseline from filename. Should have random or gpucb in file name.") if args.results_path != '.': add_baseline_to_ax(args.results_path+'/'+file, type, mean_ax, median_ax) else: add_baseline_to_ax(file, type, mean_ax, median_ax) N_plot = None for name in types: res_files = get_result_file(name, res_path) print(res_files) for N, res_files in res_files.items(): print(N) if N_plot is None: N_plot = N assert N_plot == N, \ 'You are trying to plot results with different N values. Please \ check that all results on the results path have the same N value' all_L_steps = {} for res_file in res_files: regret_results = util.read_from_file(res_file, verbose=False) Ls = sorted(regret_results.keys()) for L in Ls: L_steps = list(itertools.chain.from_iterable([regret_results[L][model] \ for model in regret_results[L].keys()])) if L not in all_L_steps: all_L_steps[L] = L_steps else: all_L_steps[L] += L_steps Ls = sorted(all_L_steps.keys()) mean_steps = [np.mean(all_L_steps[L]) for L in Ls] std_dev_steps = [np.std(all_L_steps[L]) for L in Ls] median_steps = [np.median(all_L_steps[L]) for L in Ls] q25_steps = [np.quantile(all_L_steps[L], 0.25) for L in Ls] q75_steps = [np.quantile(all_L_steps[L], 0.75) for L in Ls] med_bot, med_mid, med_top = q25_steps, median_steps, q75_steps # Quantiles mean_bot, mean_mid, mean_top = np.subtract(mean_steps, std_dev_steps), \ mean_steps, \ np.add(mean_steps, std_dev_steps) # Standard Deviation for (bot, mid, top, type, ax) in ((med_bot, med_mid, med_top, 'Median Interactions', mean_ax),\ (mean_bot, mean_mid, mean_top, 'Mean Interactions', median_ax)): for plot_type in plot_info: if plot_type in name: plot_params = plot_info[plot_type] ax.plot(Ls, mid, c=plot_params[0], label=plot_params[1]) ax.fill_between(Ls, bot, top, facecolor=plot_params[0], alpha=0.2) ax.set_ylim(top=110, bottom=0) ax.set_xlim(left=0.0, right=100.0) ax.legend() ax.set_xlabel('L') ax.set_ylabel(type) ax.set_title('Interactions to Success on N=%s Mechanisms' % N)
args = parser.parse_args() if args.debug: import pdb pdb.set_trace() # remake dirs (don't want to overwrite data) model_dir = './' + args.save_dir + '/torch_models/' runs_dir = './' + args.save_dir + '/runs' os.makedirs('./' + args.save_dir + '/') os.makedirs(model_dir) os.makedirs(runs_dir) # make tensorboard writer writer = SummaryWriter(runs_dir) all_results = util.read_from_file(args.data_fname) for L in range(args.L_min, args.L_max + 1, args.L_step): results = [] for L_results in all_results[0:L]: results += L_results[0:args.M] model_fname = model_dir + 'model_' + str(L) + 'L_' + str(args.M) + 'M' train_eval(args, args.hdim, args.batch_size, args.pviz, results, model_fname, writer) writer.close() # save run params to text file in models dir import git repo = git.Repo(search_parent_directories=True) branch = repo.active_branch git_hash = repo.head.object.hexsha