def make_regret_T_plots(res_files):
for (T,N), res_file in res_files.items():
if T in plt_axes:
pass
else:
_, median_ax = plt.subplots()
_, mean_ax = plt.subplots()
_, succ_ax = plt.subplots()
plt_axes[T] = [median_ax, mean_ax, succ_ax]
regret_results = util.read_from_file(res_file, verbose=False)
Ls = sorted(regret_results.keys())
mean_regrets = []
median_regrets = []
std_dev_regrets = []
q25_regrets = []
q75_regrets = []
prob_successes = []
std_successes = []
for L in Ls:
all_L_regrets = list(itertools.chain.from_iterable([regret_results[L][model] \
for model in regret_results[L].keys()]))
mean_regrets += [np.mean(all_L_regrets)]
median_regrets += [np.median(all_L_regrets)]
std_dev_regrets += [np.std(all_L_regrets)]
q25_regrets += [np.quantile(all_L_regrets, 0.25)]
q75_regrets += [np.quantile(all_L_regrets, 0.75)]
prob_success, std_success = get_success(all_L_regrets)
prob_successes += [prob_success]
std_successes += [std_success]
med_bot, med_mid, med_top = q25_regrets, median_regrets, q75_regrets # Quantiles
mean_bot, mean_mid, mean_top = np.subtract(mean_regrets, std_dev_regrets), \
mean_regrets, \
np.add(mean_regrets, std_dev_regrets) # Standard Deviation
succ_bot, succ_mid, succ_top = np.subtract(prob_successes, std_successes), \
prob_successes, \
np.add(prob_successes, std_successes) # Success
for (bot, mid, top, type, ax) in ((med_bot, med_mid, med_top, 'Median Regret', plt_axes[T][0]),\
(mean_bot, mean_mid, mean_top, 'Mean Regret', plt_axes[T][1]),\
(succ_bot, succ_mid, succ_top, '% Success', plt_axes[T][2])):
#plt.figure()
for plot_type in plot_info:
if plot_type in name:
plot_params = plot_info[plot_type]
ax.plot(Ls, mid, c=plot_params[0], label=plot_params[1])
ax.fill_between(Ls, bot, top, facecolor=plot_params[0], alpha=0.2)
ax.set_ylim(0, 1)
ax.legend()
ax.set_xlabel('L')
ax.set_ylabel(type)
ax.set_title('Evaluated on T=%s Interactions on N=%s Mechanisms' % (T, N))
def add_baseline_to_ax(fname, type, mean_ax, median_ax):
plot_params = plot_info[type]
steps = util.read_from_file(fname)
mean = [np.mean(steps)]*10
std = [np.std(steps)]*10
median = [np.median(steps)]*10
q25 = [np.quantile(steps, 0.25)]*10
q75 = [np.quantile(steps, 0.75)]*10
Ls = list(range(0,101,11))
mean_ax.plot(Ls, mean, c=plot_params[0], label=plot_params[1])
mean_ax.fill_between(Ls, np.subtract(mean, std), np.add(mean,std), facecolor=plot_params[0], alpha=0.2)
median_ax.plot(Ls, median, c=plot_params[0], label=plot_params[1])
median_ax.fill_between(Ls, q25, q75, facecolor=plot_params[0], alpha=0.2)
def __correct_box(arm_file, eng_file, out_file):
lines = []
lines1 = util.read_from_file(arm_file);
lines1 = ('\n'.join(lines1)).split();
lines2 = util.read_from_file(eng_file);
content = ''
if lines1 and lines2 and len(lines1) is len(lines2):
for i in range (0,len(lines1)):
if not (len(lines1[i]) == (2,1)):
try:
lines2[i] = lines2[i][lines2[i].index(' '):len(lines2[i])]
content = content + lines1[i] + lines2[i]
except ValueError:
print(bcolors.FAIL + ' Incorrect box file coordinates ... ' + bcolors.ENDC)
return
else:
print(bcolors.FAIL + ' Armenian file content is incorrect ... ' + bcolors.ENDC)
return
util.write_in_file(out_file, content)
print (bcolors.OKGREEN + 'Done... \n' + 'Created ' + out_file + ' file ... ' + bcolors.ENDC)
else:
raise NotFound('Sizes of the ' + arm_file + ' and ' + eng_file + ' files are not equal ... or files are empty')
def __generate_and_correct_box_files(size_file, font_file):
__copy_font_files(PATH_FONTS_DIR)
fonts = util.read_from_file(font_file)
sizes = util.read_json_from_file(size_file)
if not (fonts is None or sizes is None):
for size in sizes:
try:
size_lpi = size['lpi']
size_cpi = size['cpi']
except KeyError, e:
print(bcolors.WARNING + 'Warning: size file content ... ' + bcolors.ENDC)
return
for font in fonts:
__generate_and_correct_box_file(size_lpi, size_cpi, font)
def get_bb_dataset(bb_fname, n_bbs, mech_types, max_mech, urdf_num):
# Create a dataset of busyboxes.
if bb_fname == '' or bb_fname is None:
print('Creating Busyboxes.')
mech_classes = []
for mech_type in mech_types:
if mech_type == 'slider': mech_classes.append(Slider)
if mech_type == 'door': mech_classes.append(Door)
bb_dataset = []
for _ in range(n_bbs):
# TODO: i think there is a bug here...
bb = BusyBox.generate_random_busybox(max_mech=max_mech,
mech_types=mech_classes,
urdf_tag=urdf_num)
mechanism_params = bb._mechanisms[0].get_mechanism_tuple()
image_data, gripper = setup_env(bb, False, False)
bb_dataset.append([util.Result(None, mechanism_params, None, None, None,
None, image_data, None)])
print('BusyBoxes created.')
else:
# Load in a file with predetermined BusyBoxes.
bb_dataset = read_from_file(bb_fname)[:n_bbs]
return bb_dataset
def add_points_to_saved_ax(explr_p, sample_points={}):
fname = 'mean_fig.pickle'
fig, axes, all_param_data, all_angular_params, all_linear_params = util.read_from_file(
fname)
policy_type = 'Prismatic'
curr_ax_i = 0
for angular_param in all_angular_params:
if not angular_param.varied:
continue
for linear_param in all_linear_params:
if not linear_param.varied:
continue
# TODO: only works for up to 2 other_params (will have to figure out new
# visualization past that)
plt.suptitle(policy_type + ' mean fn:' + angular_param.param_name + \
' vs ' + linear_param.param_name)
# bin the other param values
all_other_params = [
PlotData(param_name, param_data.varied, param_data.bounds)
for (param_name, param_data) in all_param_data.items()
if param_name != angular_param.param_name
and param_name != linear_param.param_name and param_data.varied
]
subplot_inds_and_vals = {}
for other_param in all_other_params:
subplot_inds_and_vals[other_param.param_name] = \
list(enumerate(np.linspace(*other_param.range, N_BINS)))
if len(subplot_inds_and_vals) == 0:
n_rows = 1
n_cols = 1
else:
keys = list(subplot_inds_and_vals.keys())
if len(subplot_inds_and_vals) == 1:
n_rows = 1
n_cols = len(subplot_inds_and_vals[keys[0]])
elif len(subplot_inds_and_vals) == 2:
n_rows = len(subplot_inds_and_vals[keys[0]])
n_cols = len(subplot_inds_and_vals[keys[1]])
for single_subplot_inds_and_vals in itertools.product(
*subplot_inds_and_vals.values()):
if len(single_subplot_inds_and_vals) == 0:
subplot_num = 1
else:
row_col = [o[0] + 1 for o in single_subplot_inds_and_vals]
if len(row_col) == 1:
subplot_num = 1 * row_col[0]
else:
subplot_num = reduce(lambda x, y: n_cols * (x - 1) + y,
row_col)
# make polar subplot of mean function
axes[curr_ax_i].set_title('\n'.join([
str(all_other_params[other_param_i].param_name) + ' = ' +
str("%.2f" % other_val)
for other_param_i, (
subplot_i,
other_val) in enumerate(single_subplot_inds_and_vals)
]),
fontsize=10)
# only add points to subplot that are close to this subplot "bin"
plot_points = []
pt_colors = sample_points[policy_type]
if subplot_inds_and_vals == {}:
plot_points = [(get_plot_point(x, angular_param.param_name,
linear_param.param_name,
all_param_data, policy_type), c)
for (x, c) in pt_colors]
else:
for pt, color in pt_colors:
keep_point = True
for other_param_name, (subplot_dim, subplot_val) in \
zip(subplot_inds_and_vals, single_subplot_inds_and_vals):
all_subplot_vals = np.array([
inds_and_vals[1] for inds_and_vals in
subplot_inds_and_vals[other_param_name]
])
point_param_val = pt[x_inds[other_param_name]]
closest_index = (np.abs(all_subplot_vals -
point_param_val)).argmin()
if all_subplot_vals[closest_index] == subplot_val:
keep_point = keep_point and True
else:
keep_point = keep_point and False
if keep_point:
plot_points.append(
(get_plot_point(pt, angular_param.param_name,
linear_param.param_name,
all_param_data,
policy_type), color))
add_points(axes[curr_ax_i], plot_points)
curr_ax_i += 1
#plt.show()
fig.savefig('voo_testing/omega_%f.png' % explr_p)
def make_regret_plots(types, res_path, nonCPP):
_, median_ax = plt.subplots()
_, mean_ax = plt.subplots()
# add nonCPP baselines to plots
if nonCPP:
for file in nonCPP:
if 'random' in file:
type = 'noncpp_random'
elif 'gpucb' in file:
type = 'noncpp_gpucb'
else:
print("could not detect data type of nonCPP baseline from filename. Should have random or gpucb in file name.")
if args.results_path != '.':
add_baseline_to_ax(args.results_path+'/'+file, type, mean_ax, median_ax)
else:
add_baseline_to_ax(file, type, mean_ax, median_ax)
N_plot = None
for name in types:
res_files = get_result_file(name, res_path)
print(res_files)
for N, res_files in res_files.items():
print(N)
if N_plot is None:
N_plot = N
assert N_plot == N, \
'You are trying to plot results with different N values. Please \
check that all results on the results path have the same N value'
all_L_steps = {}
for res_file in res_files:
regret_results = util.read_from_file(res_file, verbose=False)
Ls = sorted(regret_results.keys())
for L in Ls:
L_steps = list(itertools.chain.from_iterable([regret_results[L][model] \
for model in regret_results[L].keys()]))
if L not in all_L_steps:
all_L_steps[L] = L_steps
else:
all_L_steps[L] += L_steps
Ls = sorted(all_L_steps.keys())
mean_steps = [np.mean(all_L_steps[L]) for L in Ls]
std_dev_steps = [np.std(all_L_steps[L]) for L in Ls]
median_steps = [np.median(all_L_steps[L]) for L in Ls]
q25_steps = [np.quantile(all_L_steps[L], 0.25) for L in Ls]
q75_steps = [np.quantile(all_L_steps[L], 0.75) for L in Ls]
med_bot, med_mid, med_top = q25_steps, median_steps, q75_steps # Quantiles
mean_bot, mean_mid, mean_top = np.subtract(mean_steps, std_dev_steps), \
mean_steps, \
np.add(mean_steps, std_dev_steps) # Standard Deviation
for (bot, mid, top, type, ax) in ((med_bot, med_mid, med_top, 'Median Interactions', mean_ax),\
(mean_bot, mean_mid, mean_top, 'Mean Interactions', median_ax)):
for plot_type in plot_info:
if plot_type in name:
plot_params = plot_info[plot_type]
ax.plot(Ls, mid, c=plot_params[0], label=plot_params[1])
ax.fill_between(Ls, bot, top, facecolor=plot_params[0], alpha=0.2)
ax.set_ylim(top=110, bottom=0)
ax.set_xlim(left=0.0, right=100.0)
ax.legend()
ax.set_xlabel('L')
ax.set_ylabel(type)
ax.set_title('Interactions to Success on N=%s Mechanisms' % N)
args = parser.parse_args()
if args.debug:
import pdb
pdb.set_trace()
# remake dirs (don't want to overwrite data)
model_dir = './' + args.save_dir + '/torch_models/'
runs_dir = './' + args.save_dir + '/runs'
os.makedirs('./' + args.save_dir + '/')
os.makedirs(model_dir)
os.makedirs(runs_dir)
# make tensorboard writer
writer = SummaryWriter(runs_dir)
all_results = util.read_from_file(args.data_fname)
for L in range(args.L_min, args.L_max + 1, args.L_step):
results = []
for L_results in all_results[0:L]:
results += L_results[0:args.M]
model_fname = model_dir + 'model_' + str(L) + 'L_' + str(args.M) + 'M'
train_eval(args, args.hdim, args.batch_size, args.pviz, results,
model_fname, writer)
writer.close()
# save run params to text file in models dir
import git
repo = git.Repo(search_parent_directories=True)
branch = repo.active_branch
git_hash = repo.head.object.hexsha
