def _eval_checkpoint(
self,
checkpoint_path: str,
writer: TensorboardWriter,
checkpoint_index: int = 0,
) -> None:
r"""Evaluates a single checkpoint.
Args:
checkpoint_path: path of checkpoint
writer: tensorboard writer object for logging to tensorboard
checkpoint_index: index of cur checkpoint for logging
Returns:
None
"""
config = self.config
config.defrost()
config.TASK_CONFIG.DATASET.SPLIT = self.config.EVAL.SPLIT
config.freeze()
eqa_cnn_pretrain_dataset = EQACNNPretrainDataset(config, mode="val")
eval_loader = DataLoader(
eqa_cnn_pretrain_dataset,
batch_size=config.IL.EQACNNPretrain.batch_size,
shuffle=False,
)
logger.info("[ eval_loader has {} samples ]".format(
len(eqa_cnn_pretrain_dataset)))
model = MultitaskCNN()
state_dict = torch.load(checkpoint_path)
model.load_state_dict(state_dict)
model.to(self.device).eval()
depth_loss = torch.nn.SmoothL1Loss()
ae_loss = torch.nn.SmoothL1Loss()
seg_loss = torch.nn.CrossEntropyLoss()
t = 0
avg_loss = 0.0
avg_l1 = 0.0
avg_l2 = 0.0
avg_l3 = 0.0
with torch.no_grad():
for batch in eval_loader:
t += 1
idx, gt_rgb, gt_depth, gt_seg = batch
gt_rgb = gt_rgb.to(self.device)
gt_depth = gt_depth.to(self.device)
gt_seg = gt_seg.to(self.device)
pred_seg, pred_depth, pred_rgb = model(gt_rgb)
l1 = seg_loss(pred_seg, gt_seg.long())
l2 = ae_loss(pred_rgb, gt_rgb)
l3 = depth_loss(pred_depth, gt_depth)
loss = l1 + (10 * l2) + (10 * l3)
avg_loss += loss.item()
avg_l1 += l1.item()
avg_l2 += l2.item()
avg_l3 += l3.item()
if t % config.LOG_INTERVAL == 0:
logger.info(
"[ Iter: {}; loss: {:.3f} ]".format(t, loss.item()), )
if (config.EVAL_SAVE_RESULTS
and t % config.EVAL_SAVE_RESULTS_INTERVAL == 0):
result_id = "ckpt_{}_{}".format(checkpoint_index,
idx[0].item())
result_path = os.path.join(self.config.RESULTS_DIR,
result_id)
self._save_results(
gt_rgb,
pred_rgb,
gt_seg,
pred_seg,
gt_depth,
pred_depth,
result_path,
)
avg_loss /= len(eval_loader)
avg_l1 /= len(eval_loader)
avg_l2 /= len(eval_loader)
avg_l3 /= len(eval_loader)
writer.add_scalar("avg val total loss", avg_loss, checkpoint_index)
writer.add_scalars(
"avg val individual_losses",
{
"seg_loss": avg_l1,
"ae_loss": avg_l2,
"depth_loss": avg_l3
},
checkpoint_index,
)
logger.info("[ Average loss: {:.3f} ]".format(avg_loss))
logger.info("[ Average seg loss: {:.3f} ]".format(avg_l1))
logger.info("[ Average autoencoder loss: {:.4f} ]".format(avg_l2))
logger.info("[ Average depthloss: {:.4f} ]".format(avg_l3))
def _eval_checkpoint(
self,
checkpoint_path: str,
writer: TensorboardWriter,
checkpoint_index: int = 0,
) -> None:
r"""Evaluates a single checkpoint.
Args:
checkpoint_path: path of checkpoint
writer: tensorboard writer object for logging to tensorboard
checkpoint_index: index of cur checkpoint for logging
Returns:
None
"""
# Map location CPU is almost always better than mapping to a CUDA device.
ckpt_dict = self.load_checkpoint(checkpoint_path, map_location="cpu")
if self.config.EVAL.USE_CKPT_CONFIG:
config = self._setup_eval_config(ckpt_dict["config"])
else:
config = self.config.clone()
ppo_cfg = config.RL.PPO
ans_cfg = config.RL.ANS
config.defrost()
config.TASK_CONFIG.DATASET.SPLIT = config.EVAL.SPLIT
config.freeze()
self.envs = construct_envs(config, get_env_class(config.ENV_NAME))
self._setup_actor_critic_agent(ppo_cfg, ans_cfg)
# Convert the state_dict of mapper_agent to mapper
mapper_dict = {
k.replace("mapper.", ""): v
for k, v in ckpt_dict["mapper_state_dict"].items()
}
# Converting the state_dict of local_agent to just the local_policy.
local_dict = {
k.replace("actor_critic.", ""): v
for k, v in ckpt_dict["local_state_dict"].items()
}
# Strict = False is set to ignore to handle the case where
# pose_estimator is not required.
self.mapper.load_state_dict(mapper_dict, strict=False)
self.local_actor_critic.load_state_dict(local_dict)
# Set models to evaluation
self.mapper.eval()
self.local_actor_critic.eval()
number_of_eval_episodes = self.config.TEST_EPISODE_COUNT
if number_of_eval_episodes == -1:
number_of_eval_episodes = sum(self.envs.number_of_episodes)
else:
total_num_eps = sum(self.envs.number_of_episodes)
if total_num_eps < number_of_eval_episodes:
logger.warn(
f"Config specified {number_of_eval_episodes} eval episodes"
", dataset only has {total_num_eps}."
)
logger.warn(f"Evaluating with {total_num_eps} instead.")
number_of_eval_episodes = total_num_eps
M = ans_cfg.overall_map_size
V = ans_cfg.MAPPER.map_size
s = ans_cfg.MAPPER.map_scale
imH, imW = ans_cfg.image_scale_hw
assert (
self.envs.num_envs == 1
), "Number of environments needs to be 1 for evaluation"
# Define metric accumulators
# Navigation metrics
navigation_metrics = {
"success_rate": Metric(),
"spl": Metric(),
"distance_to_goal": Metric(),
"time": Metric(),
"softspl": Metric(),
}
per_difficulty_navigation_metrics = {
"easy": {
"success_rate": Metric(),
"spl": Metric(),
"distance_to_goal": Metric(),
"time": Metric(),
"softspl": Metric(),
},
"medium": {
"success_rate": Metric(),
"spl": Metric(),
"distance_to_goal": Metric(),
"time": Metric(),
"softspl": Metric(),
},
"hard": {
"success_rate": Metric(),
"spl": Metric(),
"distance_to_goal": Metric(),
"time": Metric(),
"softspl": Metric(),
},
}
times_per_episode = deque()
times_per_step = deque()
# Define a simple function to return episode difficulty based on
# the geodesic distance
def classify_difficulty(gd):
if gd < 5.0:
return "easy"
elif gd < 10.0:
return "medium"
else:
return "hard"
eval_start_time = time.time()
# Reset environments only for the very first batch
observations = self.envs.reset()
for ep in range(number_of_eval_episodes):
# ============================== Reset agent ==============================
# Reset agent states
state_estimates = {
"pose_estimates": torch.zeros(self.envs.num_envs, 3).to(self.device),
"map_states": torch.zeros(self.envs.num_envs, 2, M, M).to(self.device),
"recurrent_hidden_states": torch.zeros(
1, self.envs.num_envs, ans_cfg.LOCAL_POLICY.hidden_size
).to(self.device),
}
# Reset ANS states
self.ans_net.reset()
self.not_done_masks = torch.zeros(self.envs.num_envs, 1, device=self.device)
self.prev_actions = torch.zeros(self.envs.num_envs, 1, device=self.device)
self.prev_batch = None
self.ep_time = torch.zeros(self.envs.num_envs, 1, device=self.device)
# =========================== Episode loop ================================
ep_start_time = time.time()
current_episodes = self.envs.current_episodes()
for ep_step in range(self.config.T_MAX):
step_start_time = time.time()
# ============================ Action step ============================
batch = self._prepare_batch(observations)
if self.prev_batch is None:
self.prev_batch = copy.deepcopy(batch)
prev_pose_estimates = state_estimates["pose_estimates"]
with torch.no_grad():
(
_,
_,
mapper_outputs,
local_policy_outputs,
state_estimates,
) = self.ans_net.act(
batch,
self.prev_batch,
state_estimates,
self.ep_time,
self.not_done_masks,
deterministic=ans_cfg.LOCAL_POLICY.deterministic_flag,
)
actions = local_policy_outputs["actions"]
# Make masks not done till reset (end of episode)
self.not_done_masks = torch.ones(
self.envs.num_envs, 1, device=self.device
)
self.prev_actions.copy_(actions)
if ep_step == 0:
state_estimates["pose_estimates"].copy_(prev_pose_estimates)
self.ep_time += 1
# Update prev batch
for k, v in batch.items():
self.prev_batch[k].copy_(v)
# Remap actions from exploration to navigation agent.
actions_rmp = self._remap_actions(actions)
# =========================== Environment step ========================
outputs = self.envs.step([a[0].item() for a in actions_rmp])
observations, _, dones, infos = [list(x) for x in zip(*outputs)]
times_per_step.append(time.time() - step_start_time)
# ============================ Process metrics ========================
if dones[0]:
times_per_episode.append(time.time() - ep_start_time)
mins_per_episode = np.mean(times_per_episode).item() / 60.0
eta_completion = mins_per_episode * (
number_of_eval_episodes - ep - 1
)
secs_per_step = np.mean(times_per_step).item()
for i in range(self.envs.num_envs):
episode_id = int(current_episodes[i].episode_id)
curr_metrics = {
"spl": infos[i]["spl"],
"softspl": infos[i]["softspl"],
"success_rate": infos[i]["success"],
"time": ep_step + 1,
"distance_to_goal": infos[i]["distance_to_goal"],
}
# Estimate difficulty of episode
episode_difficulty = classify_difficulty(
current_episodes[i].info["geodesic_distance"]
)
for k, v in curr_metrics.items():
navigation_metrics[k].update(v, 1.0)
per_difficulty_navigation_metrics[episode_difficulty][
k
].update(v, 1.0)
logger.info(f"====> {ep}/{number_of_eval_episodes} done")
for k, v in curr_metrics.items():
logger.info(f"{k:25s} : {v:10.3f}")
logger.info("{:25s} : {:10d}".format("episode_id", episode_id))
logger.info(f"Time per episode: {mins_per_episode:.3f} mins")
logger.info(f"Time per step: {secs_per_step:.3f} secs")
logger.info(f"ETA: {eta_completion:.3f} mins")
# For navigation, terminate episode loop when dones is called
break
# done-for
if checkpoint_index == 0:
try:
eval_ckpt_idx = self.config.EVAL_CKPT_PATH_DIR.split("/")[-1].split(
"."
)[1]
logger.add_filehandler(
f"{self.config.TENSORBOARD_DIR}/navigation_results_ckpt_final_{eval_ckpt_idx}.txt"
)
except:
logger.add_filehandler(
f"{self.config.TENSORBOARD_DIR}/navigation_results_ckpt_{checkpoint_index}.txt"
)
else:
logger.add_filehandler(
f"{self.config.TENSORBOARD_DIR}/navigation_results_ckpt_{checkpoint_index}.txt"
)
logger.info(
f"======= Evaluating over {number_of_eval_episodes} episodes ============="
)
logger.info(f"=======> Navigation metrics")
for k, v in navigation_metrics.items():
logger.info(f"{k}: {v.get_metric():.3f}")
writer.add_scalar(f"navigation/{k}", v.get_metric(), checkpoint_index)
for diff, diff_metrics in per_difficulty_navigation_metrics.items():
logger.info(f"=============== {diff:^10s} metrics ==============")
for k, v in diff_metrics.items():
logger.info(f"{k}: {v.get_metric():.3f}")
writer.add_scalar(
f"{diff}_navigation/{k}", v.get_metric(), checkpoint_index
)
total_eval_time = (time.time() - eval_start_time) / 60.0
logger.info(f"Total evaluation time: {total_eval_time:.3f} mins")
self.envs.close()
def _eval_checkpoint(self,
checkpoint_path: str,
writer: TensorboardWriter,
checkpoint_index: int = 0) -> None:
r"""Evaluates a single checkpoint. Assumes episode IDs are unique.
Args:
checkpoint_path: path of checkpoint
writer: tensorboard writer object for logging to tensorboard
checkpoint_index: index of cur checkpoint for logging
Returns:
None
"""
logger.info(f"checkpoint_path: {checkpoint_path}")
if self.config.EVAL.USE_CKPT_CONFIG:
config = self._setup_eval_config(
self.load_checkpoint(checkpoint_path,
map_location="cpu")["config"])
else:
config = self.config.clone()
config.defrost()
config.TASK_CONFIG.DATASET.SPLIT = config.EVAL.SPLIT
config.TASK_CONFIG.TASK.NDTW.SPLIT = config.EVAL.SPLIT
config.TASK_CONFIG.TASK.SDTW.SPLIT = config.EVAL.SPLIT
config.TASK_CONFIG.ENVIRONMENT.ITERATOR_OPTIONS.SHUFFLE = False
config.TASK_CONFIG.ENVIRONMENT.ITERATOR_OPTIONS.MAX_SCENE_REPEAT_STEPS = -1
if len(config.VIDEO_OPTION) > 0:
config.defrost()
config.TASK_CONFIG.TASK.MEASUREMENTS.append("TOP_DOWN_MAP")
config.TASK_CONFIG.TASK.MEASUREMENTS.append("COLLISIONS")
config.freeze()
# setup agent
self.envs = construct_envs_auto_reset_false(
config, get_env_class(config.ENV_NAME))
self.device = (torch.device("cuda", config.TORCH_GPU_ID)
if torch.cuda.is_available() else torch.device("cpu"))
self._setup_actor_critic_agent(config.MODEL, True, checkpoint_path)
observations = self.envs.reset()
observations = transform_obs(
observations, config.TASK_CONFIG.TASK.INSTRUCTION_SENSOR_UUID)
batch = batch_obs(observations, self.device)
eval_recurrent_hidden_states = torch.zeros(
self.actor_critic.net.num_recurrent_layers,
config.NUM_PROCESSES,
config.MODEL.STATE_ENCODER.hidden_size,
device=self.device,
)
prev_actions = torch.zeros(config.NUM_PROCESSES,
1,
device=self.device,
dtype=torch.long)
not_done_masks = torch.zeros(config.NUM_PROCESSES,
1,
device=self.device)
stats_episodes = {} # dict of dicts that stores stats per episode
if len(config.VIDEO_OPTION) > 0:
os.makedirs(config.VIDEO_DIR, exist_ok=True)
rgb_frames = [[] for _ in range(config.NUM_PROCESSES)]
self.actor_critic.eval()
while (self.envs.num_envs > 0
and len(stats_episodes) < config.EVAL.EPISODE_COUNT):
current_episodes = self.envs.current_episodes()
with torch.no_grad():
(_, actions, _,
eval_recurrent_hidden_states) = self.actor_critic.act(
batch,
eval_recurrent_hidden_states,
prev_actions,
not_done_masks,
deterministic=True,
)
# actions = batch["vln_oracle_action_sensor"].long()
prev_actions.copy_(actions)
outputs = self.envs.step([a[0].item() for a in actions])
observations, _, dones, infos = [list(x) for x in zip(*outputs)]
not_done_masks = torch.tensor(
[[0.0] if done else [1.0] for done in dones],
dtype=torch.float,
device=self.device,
)
# reset envs and observations if necessary
for i in range(self.envs.num_envs):
if len(config.VIDEO_OPTION) > 0:
frame = observations_to_image(observations[i], infos[i])
frame = append_text_to_image(
frame,
current_episodes[i].instruction.instruction_text)
rgb_frames[i].append(frame)
if not dones[i]:
continue
stats_episodes[current_episodes[i].episode_id] = infos[i]
observations[i] = self.envs.reset_at(i)[0]
prev_actions[i] = torch.zeros(1, dtype=torch.long)
if len(config.VIDEO_OPTION) > 0:
generate_video(
video_option=config.VIDEO_OPTION,
video_dir=config.VIDEO_DIR,
images=rgb_frames[i],
episode_id=current_episodes[i].episode_id,
checkpoint_idx=checkpoint_index,
metrics={
"SPL":
round(
stats_episodes[current_episodes[i].episode_id]
["spl"], 6)
},
tb_writer=writer,
)
del stats_episodes[
current_episodes[i].episode_id]["top_down_map"]
del stats_episodes[
current_episodes[i].episode_id]["collisions"]
rgb_frames[i] = []
observations = transform_obs(
observations, config.TASK_CONFIG.TASK.INSTRUCTION_SENSOR_UUID)
batch = batch_obs(observations, self.device)
envs_to_pause = []
next_episodes = self.envs.current_episodes()
for i in range(self.envs.num_envs):
if next_episodes[i].episode_id in stats_episodes:
envs_to_pause.append(i)
(
self.envs,
eval_recurrent_hidden_states,
not_done_masks,
prev_actions,
batch,
) = self._pause_envs(
envs_to_pause,
self.envs,
eval_recurrent_hidden_states,
not_done_masks,
prev_actions,
batch,
)
self.envs.close()
aggregated_stats = {}
num_episodes = len(stats_episodes)
for stat_key in next(iter(stats_episodes.values())).keys():
aggregated_stats[stat_key] = (
sum([v[stat_key]
for v in stats_episodes.values()]) / num_episodes)
split = config.TASK_CONFIG.DATASET.SPLIT
with open(f"stats_ckpt_{checkpoint_index}_{split}.json", "w") as f:
json.dump(aggregated_stats, f, indent=4)
logger.info(f"Episodes evaluated: {num_episodes}")
checkpoint_num = checkpoint_index + 1
for k, v in aggregated_stats.items():
logger.info(f"Average episode {k}: {v:.6f}")
writer.add_scalar(f"eval_{split}_{k}", v, checkpoint_num)
def _eval_checkpoint(
self,
checkpoint_path: str,
writer: TensorboardWriter,
checkpoint_index: int = 0,
) -> None:
r"""Evaluates a single checkpoint.
Args:
checkpoint_path: path of checkpoint
writer: tensorboard writer object for logging to tensorboard
checkpoint_index: index of cur checkpoint for logging
Returns:
None
"""
config = self.config
config.defrost()
config.TASK_CONFIG.DATASET.SPLIT = self.config.EVAL.SPLIT
config.freeze()
vqa_dataset = (
EQADataset(
config,
input_type="vqa",
num_frames=config.IL.VQA.num_frames,
)
.shuffle(1000)
.to_tuple(
"episode_id",
"question",
"answer",
*["{0:0=3d}.jpg".format(x) for x in range(0, 5)],
)
.map(img_bytes_2_np_array)
)
eval_loader = DataLoader(
vqa_dataset, batch_size=config.IL.VQA.batch_size
)
logger.info("eval_loader has {} samples".format(len(vqa_dataset)))
q_vocab_dict, ans_vocab_dict = vqa_dataset.get_vocab_dicts()
model_kwargs = {
"q_vocab": q_vocab_dict.word2idx_dict,
"ans_vocab": ans_vocab_dict.word2idx_dict,
"eqa_cnn_pretrain_ckpt_path": config.EQA_CNN_PRETRAIN_CKPT_PATH,
}
model = VqaLstmCnnAttentionModel(**model_kwargs)
state_dict = torch.load(
checkpoint_path, map_location={"cuda:0": "cpu"}
)
model.load_state_dict(state_dict)
lossFn = torch.nn.CrossEntropyLoss()
t = 0
avg_loss = 0.0
avg_accuracy = 0.0
avg_mean_rank = 0.0
avg_mean_reciprocal_rank = 0.0
model.eval()
model.cnn.eval()
model.to(self.device)
metrics = VqaMetric(
info={"split": "val"},
metric_names=[
"loss",
"accuracy",
"mean_rank",
"mean_reciprocal_rank",
],
log_json=os.path.join(config.OUTPUT_LOG_DIR, "eval.json"),
)
with torch.no_grad():
for batch in eval_loader:
t += 1
episode_ids, questions, answers, frame_queue = batch
questions = questions.to(self.device)
answers = answers.to(self.device)
frame_queue = frame_queue.to(self.device)
scores, _ = model(frame_queue, questions)
loss = lossFn(scores, answers)
accuracy, ranks = metrics.compute_ranks(
scores.data.cpu(), answers
)
metrics.update([loss.item(), accuracy, ranks, 1.0 / ranks])
(
metrics_loss,
accuracy,
mean_rank,
mean_reciprocal_rank,
) = metrics.get_stats(mode=0)
avg_loss += metrics_loss
avg_accuracy += accuracy
avg_mean_rank += mean_rank
avg_mean_reciprocal_rank += mean_reciprocal_rank
if t % config.LOG_INTERVAL == 0:
logger.info(metrics.get_stat_string(mode=0))
metrics.dump_log()
if (
config.EVAL_SAVE_RESULTS
and t % config.EVAL_SAVE_RESULTS_INTERVAL == 0
):
self._save_vqa_results(
checkpoint_index,
episode_ids,
questions,
frame_queue,
scores,
answers,
q_vocab_dict,
ans_vocab_dict,
)
num_batches = math.ceil(len(vqa_dataset) / config.IL.VQA.batch_size)
avg_loss /= num_batches
avg_accuracy /= num_batches
avg_mean_rank /= num_batches
avg_mean_reciprocal_rank /= num_batches
writer.add_scalar("avg val loss", avg_loss, checkpoint_index)
writer.add_scalar("avg val accuracy", avg_accuracy, checkpoint_index)
writer.add_scalar("avg val mean rank", avg_mean_rank, checkpoint_index)
writer.add_scalar(
"avg val mean reciprocal rank",
avg_mean_reciprocal_rank,
checkpoint_index,
)
logger.info("Average loss: {:.2f}".format(avg_loss))
logger.info("Average accuracy: {:.2f}".format(avg_accuracy))
logger.info("Average mean rank: {:.2f}".format(avg_mean_rank))
logger.info(
"Average mean reciprocal rank: {:.2f}".format(
avg_mean_reciprocal_rank
)
)