def get_viz_dir(env_id, seg_idx=None):
run_name = ps.CURRENT_RUN
import rollout.run_metadata as md
instr_idx = md.CUSTOM_INSTR_NO
vizdir = get_results_dir(run_name) + "/viz/" + str(env_id) + ("_" + str(seg_idx) if seg_idx is not None else "") + "_" + str(instr_idx) + "/"
os.makedirs(vizdir, exist_ok=True)
return vizdir
def evaluate_saved_rollouts():
params = P.get_current_parameters()
setup = params["Setup"]
model_name = setup["model"]
run_name = setup["run_name"]
eval_dname = get_eval_tmp_dataset_name(model_name, run_name)
eval_envs = set(list(sorted(get_correct_eval_env_id_list())))
rollouts = load_multiple_env_data(eval_dname)
present_envs = set(
[rollout[0]["env_id"] for rollout in rollouts if len(rollout) > 0])
missing_envs = eval_envs - present_envs
logdir = get_results_dir(run_name)
if len(missing_envs) > 0:
print(f"Warning! {len(missing_envs)} envs missing: {missing_envs}")
#sys.exit(1)
log("", logdir)
log(
"--------------------------------------------------------------------------------------------",
logdir)
log(f"Evaluating rollouts for run {run_name}", logdir)
log(f" using dataset {eval_dname}", logdir)
log(f" missing envs {missing_envs}", logdir)
log(
"--------------------------------------------------------------------------------------------",
logdir)
evaler1 = DataEvalNL(setup["run_name"] + "1-1",
save_images=False,
entire_trajectory=False,
aug_len=1)
evaler1.evaluate_dataset(rollouts)
results1 = evaler1.get_results()
evaler2 = DataEvalNL(setup["run_name"] + "2-2",
save_images=False,
entire_trajectory=False,
aug_len=2)
evaler2.evaluate_dataset(rollouts)
results2 = evaler2.get_results()
evalerf = DataEvalNL(setup["run_name"] + "1-2",
save_images=True,
entire_trajectory=False)
evalerf.evaluate_dataset(rollouts)
resultsf = evalerf.get_results()
log(f"Results 1-1:{results1}", logdir)
log(f"Results 2-2:{results2}", logdir)
log(f"Results 1-2:{resultsf}", logdir)
log(f" -- END EVALUATION FOR {run_name}-- ", logdir)
log(
"--------------------------------------------------------------------------------------------",
logdir)
def save_results_extra_image(run_name, env_id, set_id, seg_id, name, image_np):
results_dir = get_results_dir(run_name, makedir=True)
extra_dir = os.path.join(results_dir, "extra")
os.makedirs(extra_dir, exist_ok=True)
img_filename = str(env_id) + "_" + str(set_id) + "_" + str(seg_id) + "_extra-" + name + ".jpg"
img_path = os.path.join(extra_dir, img_filename)
imsave(img_path, image_np)
def save_results_extra_image(run_name, env_id, set_id, seg_id, name, image_np, extra=True):
results_dir = get_results_dir(run_name, makedir=True)
if extra:
extra_dir = os.path.join(results_dir, "extra")
else:
extra_dir = results_dir
os.makedirs(extra_dir, exist_ok=True)
img_filename = str(env_id) + "_" + str(set_id) + "_" + str(seg_id) + f"{'_extra-' if extra else ''}" + name + ".jpg"
img_path = os.path.join(extra_dir, img_filename)
imsave(img_path, image_np)
def get_viz_dir_for_rollout():
run_name = ps.get_current_run_name()
import rollout.run_metadata as md
instr_idx = md.CUSTOM_INSTR_NO
env_id = md.ENV_ID
seg_idx = md.SEG_IDX
real_drone = md.REAL_DRONE
vizdir = os.path.join(
get_results_dir(run_name),
f"viz_{'real' if real_drone else 'sim'}/{env_id}{'_' + str(seg_idx) if seg_idx is not None else ''}_{instr_idx}/"
)
os.makedirs(vizdir, exist_ok=True)
return vizdir
def save_results_extra_gif(run_name, env_id, set_id, seg_id, name, image_list):
results_dir = get_results_dir(run_name, makedir=True)
extra_dir = os.path.join(results_dir, "extra")
os.makedirs(extra_dir, exist_ok=True)
if len(image_list) == 0:
print ("Empty image list: Not saving GIF!")
return
img_filename = str(env_id) + "_" + str(set_id) + "_" + str(seg_id) + "_extra-" + name + ".gif"
img_path = os.path.join(extra_dir, img_filename)
image_list_filtered = [filter_for_gif(image) for image in image_list]
imageio.mimsave(img_path, image_list_filtered, fps=5.0)
def evaluate_top_down_pred():
P.initialize_experiment()
setup = P.get_current_parameters()["Setup"]
model, model_loaded = load_model()
eval_envs = get_correct_eval_env_id_list()
dataset_name = P.get_current_parameters().get("Data").get("dataset_name")
dataset = model.get_dataset(envs=eval_envs,
dataset_prefix=dataset_name,
dataset_prefix="supervised",
eval=eval)
dataloader = DataLoader(dataset,
collate_fn=dataset.collate_fn,
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=False)
total_loss = 0
count = 0
num_batches = len(dataloader)
for b, batch in enumerate(dataloader):
loss_var = model.sup_loss_on_batch(batch, eval=True, viz=True)
total_loss += loss_var.data[0]
count += 1
print("batch: " + str(b) + " / " + str(num_batches) + \
" loss: " + str(loss_var.data[0]))
avg_loss = total_loss / count
results_dir = get_results_dir(setup["run_name"])
results_json_path = get_results_path(setup["run_name"])
os.makedirs(results_dir, exist_ok=True)
viz = model.get_viz()
for key, lst in viz.items():
for i, img in enumerate(lst):
img_path = os.path.join(
results_dir, key + str(i) + "_" + setup["model"] + ".jpg")
sp.misc.imsave(img_path, img)
print("Saved image: " + img_path)
with open(results_json_path, "w") as fp:
json.dump({"loss": avg_loss}, fp)
def evaluate_rollout(self, rollout):
last_sample = rollout[-1]
env_id = last_sample["metadata"]["env_id"]
seg_idx = last_sample["metadata"]["seg_idx"]
set_idx = last_sample["metadata"]["set_idx"]
# TODO: Allow multiple instruction sets / paths per env
path = load_path(env_id)
if self.entire_trajectory:
path_end_idx = len(path) - 1
else:
# Find the segment end index
path_end_idx = self.all_i[env_id][set_idx]["instructions"][seg_idx]["end_idx"]
if path_end_idx > len(path) - 1:
path_end_idx = len(path) - 1
end_pos = np.asarray(last_sample["state"].get_pos())
target_end_pos = np.asarray(path[path_end_idx])
end_dist = np.linalg.norm(end_pos - target_end_pos)
success = end_dist < DEFAULT_PASSING_DISTANCE
if last_sample["metadata"]["pol_action"][3] > 0.5:
who_stopped = "Policy Stopped"
elif last_sample["metadata"]["ref_action"][3] > 0.5:
who_stopped = "Oracle Stopped"
else:
who_stopped = "Veered Off"
result = "Success" if success else "Fail"
texts = [who_stopped, result, "run:" + self.run_name]
print(seg_idx, result)
if self.save_images:
dir = get_results_dir(self.run_name, makedir=True)
print("Results dir: ", dir)
self.presenter.plot_paths(rollout, interactive=False, texts=texts, entire_trajectory=self.entire_trajectory)
filename = os.path.join(dir, str(env_id) + "_" + str(set_idx) + "_" + str(seg_idx))
if self.custom_instr is not None:
filename += "_" + last_sample["metadata"]["instruction"][:24] + "_" + last_sample["metadata"]["instruction"][-16:]
self.presenter.save_plot(filename)
self.save_results()
return ResultsLandmarkSide(success, end_dist)
def evaluate_rollout(self, rollout):
last_sample = rollout[-1]
env_id = last_sample["metadata"]["env_id"]
seg_idx = last_sample["metadata"]["seg_idx"]
set_idx = last_sample["metadata"]["set_idx"]
# TODO: Allow multiple templates / instructions per env
path = load_path(env_id)
end_pos = np.asarray(last_sample["state"].get_pos())
landmark_pos = self.get_landmark_pos(env_id)
target_end_pos = np.asarray(path[-1])
end_goal_dist = np.linalg.norm(end_pos - target_end_pos)
end_lm_dist = np.linalg.norm(end_pos - landmark_pos)
correct_landmark_region = end_lm_dist < LANDMARK_REGION_RADIUS
correct_quadrant = self.correct_side(rollout, env_id)
if last_sample["metadata"]["pol_action"][3] > 0.5:
who_stopped = "Policy Stopped"
elif last_sample["metadata"]["ref_action"][3] > 0.5:
who_stopped = "Oracle Stopped"
else:
who_stopped = "Veered Off"
success = correct_landmark_region and correct_quadrant
side_txt = "Correct landmark" if correct_landmark_region else "Wrong landmark"
result = "Success" if success else "Fail"
texts = [who_stopped, result, side_txt, "run:" + self.run_name]
if self.save_images:
dir = get_results_dir(self.run_name, makedir=True)
self.presenter.plot_paths(rollout, interactive=False,
texts=[]) #texts)
filename = os.path.join(
dir,
str(env_id) + "_" + str(set_idx) + "_" + str(seg_idx))
self.presenter.save_plot(filename)
self.save_results()
return ResultsLandmarkSide(success, end_goal_dist,
correct_landmark_region)
def evaluate_rollout(self, rollout):
last_sample = rollout[-1]
if "metadata" not in last_sample:
last_sample["metadata"] = last_sample
env_id = last_sample["metadata"]["env_id"]
# TEMPORARY FOR APPENDIX TABLE! REMOVE IT!
# if env_id >= 6000:
# return None
seg_idx = last_sample["metadata"]["seg_idx"]
set_idx = last_sample["metadata"]["set_idx"]
path = load_and_convert_path(env_id)
seg_ordinal = seg_idx_to_ordinal(
self.all_i[env_id][set_idx]["instructions"], seg_idx)
instr_seg = self.all_i[env_id][set_idx]["instructions"][seg_ordinal]
if self.entire_trajectory:
path_end_idx = len(path) - 1
path_start_idx = 0
else:
# Find the segment end index
path_end_idx = self.all_i[env_id][set_idx]["instructions"][
seg_ordinal]["end_idx"] + 1
path_start_idx = self.all_i[env_id][set_idx]["instructions"][
seg_ordinal]["start_idx"]
if path_end_idx > len(path) - 1:
path_end_idx = len(path) - 1
if path_end_idx < path_start_idx:
path_start_idx = path_end_idx
seg_path = path[path_start_idx:path_end_idx]
goal_visible = self.is_goal_visible(instr_seg)
self.visible_map[f"{env_id}_{seg_idx}"] = (1 if goal_visible else 0)
exec_path = np.asarray([r["state"].get_pos_2d() for r in rollout])
end_pos = np.asarray(exec_path[-1]) #["state"].get_pos_2d())
target_end_pos = np.asarray(seg_path[-1])
end_dist = np.linalg.norm(end_pos - target_end_pos)
success = end_dist < self.passing_distance
# EMD between trajectories, and EMD between start position and trajectory.
exec_path = self._filter_path(exec_path)
gt_path = self._filter_path(seg_path)
emd = self._calculate_emd(exec_path, gt_path)
stop_emd = self._calculate_emd(exec_path[0:1], gt_path)
# Success weighted by earth-mover's distance
nemd = emd / stop_emd
semd = max((1 if success else 0) * (1 - nemd), 0)
if last_sample["metadata"]["pol_action"][3] > 0.5:
who_stopped = "Policy Stopped"
elif last_sample["metadata"]["ref_action"][3] > 0.5:
who_stopped = "Oracle Stopped"
else:
who_stopped = "Veered Off"
result = "Success" if success else "Fail"
print(env_id, set_idx, seg_idx, result)
texts = [who_stopped, result, "run:" + self.run_name]
#print(seg_idx, result, semd)
if self.save_images and emd:
dir = get_results_dir(self.run_name, makedir=True)
print("Results dir: ", dir)
# TODO: Refactor this to not pull path from rollout, but provide it explicitly
self.presenter.plot_paths(
rollout,
segment_path=gt_path,
interactive=False,
texts=texts,
entire_trajectory=self.entire_trajectory,
world_size=P.get_current_parameters()["Setup"]["world_size_m"],
real_drone=P.get_current_parameters()["Setup"]["real_drone"])
filename = os.path.join(
dir,
str(env_id) + "_" + str(set_idx) + "_" + str(seg_idx))
if self.custom_instr is not None:
filename += "_" + last_sample["metadata"][
"instruction"][:24] + "_" + last_sample["metadata"][
"instruction"][-16:]
self.presenter.save_plot(filename)
#if emd:
# self.save_results()
return ResultsLandmarkSide(success=success,
end_dist=end_dist,
goal_visible=goal_visible,
emd=emd,
semd=semd,
nemd=nemd)
return doc.getvalue()
def find_file_groups(filename_list):
regexp = "[A-Za-z_]*1\.png"
first_files = list(filter(re.compile(regexp).match, filename_list))
groups = [f.split("1")[0] for f in first_files]
return groups
if __name__ == "__main__":
P.initialize_experiment(sys.argv[1])
run_name = P.get_current_parameters()["Setup"]["run_name"]
results_dir = get_results_dir(run_name=run_name)
sim_dir = os.path.join(results_dir, "viz_sim")
real_dir = os.path.join(results_dir, "viz_real")
rollouts_sim = set(os.listdir(sim_dir))
rollouts_real = set(os.listdir(real_dir))
rollouts_common = list(rollouts_real.intersection(rollouts_sim))
doc, tag, text = start_document()
for rollout in rollouts_common:
html_table_header = []
html_table_data = []
rollout_real_dir = os.path.join(real_dir, rollout)
rollout_sim_dir = os.path.join(sim_dir, rollout)
