def get_viz_dir(env_id, seg_idx=None): run_name = ps.CURRENT_RUN import rollout.run_metadata as md instr_idx = md.CUSTOM_INSTR_NO vizdir = get_results_dir(run_name) + "/viz/" + str(env_id) + ("_" + str(seg_idx) if seg_idx is not None else "") + "_" + str(instr_idx) + "/" os.makedirs(vizdir, exist_ok=True) return vizdir
def evaluate_saved_rollouts(): params = P.get_current_parameters() setup = params["Setup"] model_name = setup["model"] run_name = setup["run_name"] eval_dname = get_eval_tmp_dataset_name(model_name, run_name) eval_envs = set(list(sorted(get_correct_eval_env_id_list()))) rollouts = load_multiple_env_data(eval_dname) present_envs = set( [rollout[0]["env_id"] for rollout in rollouts if len(rollout) > 0]) missing_envs = eval_envs - present_envs logdir = get_results_dir(run_name) if len(missing_envs) > 0: print(f"Warning! {len(missing_envs)} envs missing: {missing_envs}") #sys.exit(1) log("", logdir) log( "--------------------------------------------------------------------------------------------", logdir) log(f"Evaluating rollouts for run {run_name}", logdir) log(f" using dataset {eval_dname}", logdir) log(f" missing envs {missing_envs}", logdir) log( "--------------------------------------------------------------------------------------------", logdir) evaler1 = DataEvalNL(setup["run_name"] + "1-1", save_images=False, entire_trajectory=False, aug_len=1) evaler1.evaluate_dataset(rollouts) results1 = evaler1.get_results() evaler2 = DataEvalNL(setup["run_name"] + "2-2", save_images=False, entire_trajectory=False, aug_len=2) evaler2.evaluate_dataset(rollouts) results2 = evaler2.get_results() evalerf = DataEvalNL(setup["run_name"] + "1-2", save_images=True, entire_trajectory=False) evalerf.evaluate_dataset(rollouts) resultsf = evalerf.get_results() log(f"Results 1-1:{results1}", logdir) log(f"Results 2-2:{results2}", logdir) log(f"Results 1-2:{resultsf}", logdir) log(f" -- END EVALUATION FOR {run_name}-- ", logdir) log( "--------------------------------------------------------------------------------------------", logdir)
def save_results_extra_image(run_name, env_id, set_id, seg_id, name, image_np): results_dir = get_results_dir(run_name, makedir=True) extra_dir = os.path.join(results_dir, "extra") os.makedirs(extra_dir, exist_ok=True) img_filename = str(env_id) + "_" + str(set_id) + "_" + str(seg_id) + "_extra-" + name + ".jpg" img_path = os.path.join(extra_dir, img_filename) imsave(img_path, image_np)
def save_results_extra_image(run_name, env_id, set_id, seg_id, name, image_np, extra=True): results_dir = get_results_dir(run_name, makedir=True) if extra: extra_dir = os.path.join(results_dir, "extra") else: extra_dir = results_dir os.makedirs(extra_dir, exist_ok=True) img_filename = str(env_id) + "_" + str(set_id) + "_" + str(seg_id) + f"{'_extra-' if extra else ''}" + name + ".jpg" img_path = os.path.join(extra_dir, img_filename) imsave(img_path, image_np)
def get_viz_dir_for_rollout(): run_name = ps.get_current_run_name() import rollout.run_metadata as md instr_idx = md.CUSTOM_INSTR_NO env_id = md.ENV_ID seg_idx = md.SEG_IDX real_drone = md.REAL_DRONE vizdir = os.path.join( get_results_dir(run_name), f"viz_{'real' if real_drone else 'sim'}/{env_id}{'_' + str(seg_idx) if seg_idx is not None else ''}_{instr_idx}/" ) os.makedirs(vizdir, exist_ok=True) return vizdir
def save_results_extra_gif(run_name, env_id, set_id, seg_id, name, image_list): results_dir = get_results_dir(run_name, makedir=True) extra_dir = os.path.join(results_dir, "extra") os.makedirs(extra_dir, exist_ok=True) if len(image_list) == 0: print ("Empty image list: Not saving GIF!") return img_filename = str(env_id) + "_" + str(set_id) + "_" + str(seg_id) + "_extra-" + name + ".gif" img_path = os.path.join(extra_dir, img_filename) image_list_filtered = [filter_for_gif(image) for image in image_list] imageio.mimsave(img_path, image_list_filtered, fps=5.0)
def evaluate_top_down_pred(): P.initialize_experiment() setup = P.get_current_parameters()["Setup"] model, model_loaded = load_model() eval_envs = get_correct_eval_env_id_list() dataset_name = P.get_current_parameters().get("Data").get("dataset_name") dataset = model.get_dataset(envs=eval_envs, dataset_prefix=dataset_name, dataset_prefix="supervised", eval=eval) dataloader = DataLoader(dataset, collate_fn=dataset.collate_fn, batch_size=1, shuffle=False, num_workers=1, pin_memory=False) total_loss = 0 count = 0 num_batches = len(dataloader) for b, batch in enumerate(dataloader): loss_var = model.sup_loss_on_batch(batch, eval=True, viz=True) total_loss += loss_var.data[0] count += 1 print("batch: " + str(b) + " / " + str(num_batches) + \ " loss: " + str(loss_var.data[0])) avg_loss = total_loss / count results_dir = get_results_dir(setup["run_name"]) results_json_path = get_results_path(setup["run_name"]) os.makedirs(results_dir, exist_ok=True) viz = model.get_viz() for key, lst in viz.items(): for i, img in enumerate(lst): img_path = os.path.join( results_dir, key + str(i) + "_" + setup["model"] + ".jpg") sp.misc.imsave(img_path, img) print("Saved image: " + img_path) with open(results_json_path, "w") as fp: json.dump({"loss": avg_loss}, fp)
def evaluate_rollout(self, rollout): last_sample = rollout[-1] env_id = last_sample["metadata"]["env_id"] seg_idx = last_sample["metadata"]["seg_idx"] set_idx = last_sample["metadata"]["set_idx"] # TODO: Allow multiple instruction sets / paths per env path = load_path(env_id) if self.entire_trajectory: path_end_idx = len(path) - 1 else: # Find the segment end index path_end_idx = self.all_i[env_id][set_idx]["instructions"][seg_idx]["end_idx"] if path_end_idx > len(path) - 1: path_end_idx = len(path) - 1 end_pos = np.asarray(last_sample["state"].get_pos()) target_end_pos = np.asarray(path[path_end_idx]) end_dist = np.linalg.norm(end_pos - target_end_pos) success = end_dist < DEFAULT_PASSING_DISTANCE if last_sample["metadata"]["pol_action"][3] > 0.5: who_stopped = "Policy Stopped" elif last_sample["metadata"]["ref_action"][3] > 0.5: who_stopped = "Oracle Stopped" else: who_stopped = "Veered Off" result = "Success" if success else "Fail" texts = [who_stopped, result, "run:" + self.run_name] print(seg_idx, result) if self.save_images: dir = get_results_dir(self.run_name, makedir=True) print("Results dir: ", dir) self.presenter.plot_paths(rollout, interactive=False, texts=texts, entire_trajectory=self.entire_trajectory) filename = os.path.join(dir, str(env_id) + "_" + str(set_idx) + "_" + str(seg_idx)) if self.custom_instr is not None: filename += "_" + last_sample["metadata"]["instruction"][:24] + "_" + last_sample["metadata"]["instruction"][-16:] self.presenter.save_plot(filename) self.save_results() return ResultsLandmarkSide(success, end_dist)
def evaluate_rollout(self, rollout): last_sample = rollout[-1] env_id = last_sample["metadata"]["env_id"] seg_idx = last_sample["metadata"]["seg_idx"] set_idx = last_sample["metadata"]["set_idx"] # TODO: Allow multiple templates / instructions per env path = load_path(env_id) end_pos = np.asarray(last_sample["state"].get_pos()) landmark_pos = self.get_landmark_pos(env_id) target_end_pos = np.asarray(path[-1]) end_goal_dist = np.linalg.norm(end_pos - target_end_pos) end_lm_dist = np.linalg.norm(end_pos - landmark_pos) correct_landmark_region = end_lm_dist < LANDMARK_REGION_RADIUS correct_quadrant = self.correct_side(rollout, env_id) if last_sample["metadata"]["pol_action"][3] > 0.5: who_stopped = "Policy Stopped" elif last_sample["metadata"]["ref_action"][3] > 0.5: who_stopped = "Oracle Stopped" else: who_stopped = "Veered Off" success = correct_landmark_region and correct_quadrant side_txt = "Correct landmark" if correct_landmark_region else "Wrong landmark" result = "Success" if success else "Fail" texts = [who_stopped, result, side_txt, "run:" + self.run_name] if self.save_images: dir = get_results_dir(self.run_name, makedir=True) self.presenter.plot_paths(rollout, interactive=False, texts=[]) #texts) filename = os.path.join( dir, str(env_id) + "_" + str(set_idx) + "_" + str(seg_idx)) self.presenter.save_plot(filename) self.save_results() return ResultsLandmarkSide(success, end_goal_dist, correct_landmark_region)
def evaluate_rollout(self, rollout): last_sample = rollout[-1] if "metadata" not in last_sample: last_sample["metadata"] = last_sample env_id = last_sample["metadata"]["env_id"] # TEMPORARY FOR APPENDIX TABLE! REMOVE IT! # if env_id >= 6000: # return None seg_idx = last_sample["metadata"]["seg_idx"] set_idx = last_sample["metadata"]["set_idx"] path = load_and_convert_path(env_id) seg_ordinal = seg_idx_to_ordinal( self.all_i[env_id][set_idx]["instructions"], seg_idx) instr_seg = self.all_i[env_id][set_idx]["instructions"][seg_ordinal] if self.entire_trajectory: path_end_idx = len(path) - 1 path_start_idx = 0 else: # Find the segment end index path_end_idx = self.all_i[env_id][set_idx]["instructions"][ seg_ordinal]["end_idx"] + 1 path_start_idx = self.all_i[env_id][set_idx]["instructions"][ seg_ordinal]["start_idx"] if path_end_idx > len(path) - 1: path_end_idx = len(path) - 1 if path_end_idx < path_start_idx: path_start_idx = path_end_idx seg_path = path[path_start_idx:path_end_idx] goal_visible = self.is_goal_visible(instr_seg) self.visible_map[f"{env_id}_{seg_idx}"] = (1 if goal_visible else 0) exec_path = np.asarray([r["state"].get_pos_2d() for r in rollout]) end_pos = np.asarray(exec_path[-1]) #["state"].get_pos_2d()) target_end_pos = np.asarray(seg_path[-1]) end_dist = np.linalg.norm(end_pos - target_end_pos) success = end_dist < self.passing_distance # EMD between trajectories, and EMD between start position and trajectory. exec_path = self._filter_path(exec_path) gt_path = self._filter_path(seg_path) emd = self._calculate_emd(exec_path, gt_path) stop_emd = self._calculate_emd(exec_path[0:1], gt_path) # Success weighted by earth-mover's distance nemd = emd / stop_emd semd = max((1 if success else 0) * (1 - nemd), 0) if last_sample["metadata"]["pol_action"][3] > 0.5: who_stopped = "Policy Stopped" elif last_sample["metadata"]["ref_action"][3] > 0.5: who_stopped = "Oracle Stopped" else: who_stopped = "Veered Off" result = "Success" if success else "Fail" print(env_id, set_idx, seg_idx, result) texts = [who_stopped, result, "run:" + self.run_name] #print(seg_idx, result, semd) if self.save_images and emd: dir = get_results_dir(self.run_name, makedir=True) print("Results dir: ", dir) # TODO: Refactor this to not pull path from rollout, but provide it explicitly self.presenter.plot_paths( rollout, segment_path=gt_path, interactive=False, texts=texts, entire_trajectory=self.entire_trajectory, world_size=P.get_current_parameters()["Setup"]["world_size_m"], real_drone=P.get_current_parameters()["Setup"]["real_drone"]) filename = os.path.join( dir, str(env_id) + "_" + str(set_idx) + "_" + str(seg_idx)) if self.custom_instr is not None: filename += "_" + last_sample["metadata"][ "instruction"][:24] + "_" + last_sample["metadata"][ "instruction"][-16:] self.presenter.save_plot(filename) #if emd: # self.save_results() return ResultsLandmarkSide(success=success, end_dist=end_dist, goal_visible=goal_visible, emd=emd, semd=semd, nemd=nemd)
return doc.getvalue() def find_file_groups(filename_list): regexp = "[A-Za-z_]*1\.png" first_files = list(filter(re.compile(regexp).match, filename_list)) groups = [f.split("1")[0] for f in first_files] return groups if __name__ == "__main__": P.initialize_experiment(sys.argv[1]) run_name = P.get_current_parameters()["Setup"]["run_name"] results_dir = get_results_dir(run_name=run_name) sim_dir = os.path.join(results_dir, "viz_sim") real_dir = os.path.join(results_dir, "viz_real") rollouts_sim = set(os.listdir(sim_dir)) rollouts_real = set(os.listdir(real_dir)) rollouts_common = list(rollouts_real.intersection(rollouts_sim)) doc, tag, text = start_document() for rollout in rollouts_common: html_table_header = [] html_table_data = [] rollout_real_dir = os.path.join(real_dir, rollout) rollout_sim_dir = os.path.join(sim_dir, rollout)