def _train_report(self, epoch_id):
# compute ETA
fetch_time = self.log["train"]["fetch"]
forward_time = self.log["train"]["forward"]
backward_time = self.log["train"]["backward"]
eval_time = self.log["train"]["eval"]
iter_time = self.log["train"]["iter_time"]
mean_train_time = np.mean(iter_time)
mean_est_val_time = np.mean([
fetch + forward for fetch, forward in zip(fetch_time, forward_time)
])
eta_sec = (self._total_iter["train"] - self._global_iter_id -
1) * mean_train_time
eta_sec += len(self.dataloader["val"]) * np.ceil(
self._total_iter["train"] / self.val_step) * mean_est_val_time
eta = decode_eta(eta_sec)
# print report
iter_report = self.__iter_report_template.format(
epoch_id=epoch_id + 1,
iter_id=self._global_iter_id + 1,
total_iter=self._total_iter["train"],
train_loss=round(np.mean([v for v in self.log["train"]["loss"]]),
5),
train_ref_loss=round(
np.mean([v for v in self.log["train"]["ref_loss"]]), 5),
train_lang_loss=round(
np.mean([v for v in self.log["train"]["lang_loss"]]), 5),
train_objectness_loss=round(
np.mean([v for v in self.log["train"]["objectness_loss"]]), 5),
train_vote_loss=round(
np.mean([v for v in self.log["train"]["vote_loss"]]), 5),
train_box_loss=round(
np.mean([v for v in self.log["train"]["box_loss"]]), 5),
train_lang_acc=round(
np.mean([v for v in self.log["train"]["lang_acc"]]), 5),
train_ref_acc=round(
np.mean([v for v in self.log["train"]["ref_acc"]]), 5),
train_obj_acc=round(
np.mean([v for v in self.log["train"]["obj_acc"]]), 5),
train_pos_ratio=round(
np.mean([v for v in self.log["train"]["pos_ratio"]]), 5),
train_neg_ratio=round(
np.mean([v for v in self.log["train"]["neg_ratio"]]), 5),
train_iou_rate_25=round(
np.mean([v for v in self.log["train"]["iou_rate_0.25"]]), 5),
train_iou_rate_5=round(
np.mean([v for v in self.log["train"]["iou_rate_0.5"]]), 5),
mean_fetch_time=round(np.mean(fetch_time), 5),
mean_forward_time=round(np.mean(forward_time), 5),
mean_backward_time=round(np.mean(backward_time), 5),
mean_eval_time=round(np.mean(eval_time), 5),
mean_iter_time=round(np.mean(iter_time), 5),
eta_h=eta["h"],
eta_m=eta["m"],
eta_s=eta["s"])
self._log(iter_report)
def _train_report(self, epoch_id):
# compute ETA
fetch_time = self.log["train"]["fetch"]
forward_time = self.log["train"]["forward"]
backward_time = self.log["train"]["backward"]
eval_time = self.log["train"]["eval"]
iter_time = self.log["train"]["iter_time"]
mean_train_time = np.mean(iter_time)
mean_est_val_time = np.mean([
fetch + forward for fetch, forward in zip(fetch_time, forward_time)
])
num_train_iter_left = self._total_iter[
"train"] - self._global_iter_id - 1
eta_sec = num_train_iter_left * mean_train_time
num_val_times = num_train_iter_left // self.val_step
eta_sec += len(self.dataloader["eval"]
["train"]) * num_val_times * mean_est_val_time
eta_sec += len(
self.dataloader["eval"]["val"]) * num_val_times * mean_est_val_time
eta = decode_eta(eta_sec)
# print report
iter_report = self.__iter_report_template.format(
epoch_id=epoch_id + 1,
iter_id=self._global_iter_id + 1,
total_iter=self._total_iter["train"],
train_loss=round(np.mean([v for v in self.log["train"]["loss"]]),
5),
train_cap_loss=round(
np.mean([v for v in self.log["train"]["cap_loss"]]), 5),
train_ori_loss=round(
np.mean([v for v in self.log["train"]["ori_loss"]]), 5),
train_dist_loss=round(
np.mean([v for v in self.log["train"]["dist_loss"]]), 5),
train_objectness_loss=round(
np.mean([v for v in self.log["train"]["objectness_loss"]]), 5),
train_vote_loss=round(
np.mean([v for v in self.log["train"]["vote_loss"]]), 5),
train_box_loss=round(
np.mean([v for v in self.log["train"]["box_loss"]]), 5),
train_cap_acc=round(
np.mean([v for v in self.log["train"]["cap_acc"]]), 5),
train_ori_acc=round(
np.mean([v for v in self.log["train"]["ori_acc"]]), 5),
train_obj_acc=round(
np.mean([v for v in self.log["train"]["obj_acc"]]), 5),
train_pos_ratio=round(
np.mean([v for v in self.log["train"]["pos_ratio"]]), 5),
train_neg_ratio=round(
np.mean([v for v in self.log["train"]["neg_ratio"]]), 5),
train_pred_ious=round(
np.mean([v for v in self.log["train"]["pred_ious"]]), 5),
mean_fetch_time=round(np.mean(fetch_time), 5),
mean_forward_time=round(np.mean(forward_time), 5),
mean_backward_time=round(np.mean(backward_time), 5),
mean_eval_time=round(np.mean(eval_time), 5),
mean_iter_time=round(np.mean(iter_time), 5),
eta_h=eta["h"],
eta_m=eta["m"],
eta_s=eta["s"])
self._log(iter_report)