def log_iter_stats(self, cur_epoch, cur_iter, global_iter):
"""
log the stats of the current iteration.
Args:
cur_epoch (int): the number of current epoch.
cur_iter (int): the number of current iteration in current epoch.
global_iter (int): the number of current iteration in training phase
"""
if (cur_iter + 1) % self._cfg.LOG_PERIOD != 0:
return
eta_sec = self.iter_timer.seconds() * (
self.MAX_ITERS - (cur_epoch * self.epoch_iters + cur_iter + 1))
eta = str(datetime.timedelta(seconds=int(eta_sec)))
stats = {
"_type": "train_iter",
"global_iter": "{}/{}".format(global_iter, self.MAX_ITERS),
"epoch": "{}/{}".format(cur_epoch + 1, self.MAX_EPOCH),
"iter": "{}/{}".format(cur_iter + 1, self.epoch_iters),
"time_diff": self.iter_timer.seconds(),
"eta": eta,
"loss": self.loss.get_win_median(),
"lr": self.lr,
"gpu_mem": int(np.ceil(misc.gpu_mem_usage())),
}
if not self._cfg.DATA.MULTI_LABEL:
stats["top1_err"] = self.mb_top1_err.get_win_median()
stats["top5_err"] = self.mb_top5_err.get_win_median()
logging.log_json_stats(stats)
def log_epoch_stats(self, cur_epoch):
"""
Log the stats of the current epoch.
Args:
cur_epoch (int): the number of current epoch.
"""
eta_sec = self.iter_timer.seconds() * (
self.MAX_ITERS - (cur_epoch + 1) * self.epoch_iters)
eta = str(datetime.timedelta(seconds=int(eta_sec)))
stats = {
"_type": "train_epoch",
"epoch": "{}/{}".format(cur_epoch + 1, self.MAX_EPOCH),
"time_diff": self.iter_timer.seconds(),
"eta": eta,
"lr": self.lr,
"gpu_mem": int(np.ceil(misc.gpu_mem_usage())),
}
if not self._cfg.DATA.MULTI_LABEL:
top1_err = self.num_top1_mis / self.num_samples
top5_err = self.num_top5_mis / self.num_samples
avg_loss = self.loss_total / self.num_samples
stats["top1_err"] = top1_err
stats["top5_err"] = top5_err
stats["loss"] = avg_loss
logging.log_json_stats(stats)
def finalize_metrics(self, ks=(1, 5)):
"""
Calculate and log the final ensembled metrics.
ks (tuple): list of top-k values for topk_accuracies. For example,
ks = (1, 5) correspods to top-1 and top-5 accuracy.
"""
if not all(self.clip_count == self.num_clips):
logger.warning("clip count {} ~= num clips {}".format(
", ".join([
"{}: {}".format(i, k)
for i, k in enumerate(self.clip_count.tolist())
]),
self.num_clips,
))
stats = {"split": "test_final"}
if self.multi_label:
map = get_map(self.video_preds.cpu().numpy(),
self.video_labels.cpu().numpy())
stats["map"] = map
else:
num_topks_correct = metrics.topks_correct(self.video_preds,
self.video_labels, ks)
topks = [(x / self.video_preds.size(0)) * 100.0
for x in num_topks_correct]
assert len({len(ks), len(topks)}) == 1
for k, topk in zip(ks, topks):
stats["top{}_acc".format(k)] = "{:.{prec}f}".format(topk,
prec=2)
logging.log_json_stats(stats)
def log_epoch_stats(self, cur_epoch):
"""
Log the stats of the current epoch.
Args:
cur_epoch (int): the number of current epoch.
"""
mem_usage = misc.gpu_mem_usage()
stats = {
"_type": "val_epoch",
"epoch": "{}/{}".format(cur_epoch + 1, self.MAX_EPOCH),
"time_diff": self.iter_timer.seconds(),
"gpu_mem": int(np.ceil(mem_usage)),
}
if self._cfg.DATA.MULTI_LABEL:
stats["map"] = get_map(
torch.cat(self.all_preds).cpu().numpy(),
torch.cat(self.all_labels).cpu().numpy(),
)
self.max_map = max(self.max_map, stats["map"])
stats["max_map"] = self.max_map
else:
top1_err = self.num_top1_mis / self.num_samples
top5_err = self.num_top5_mis / self.num_samples
self.min_top1_err = min(self.min_top1_err, top1_err)
self.min_top5_err = min(self.min_top5_err, top5_err)
stats["top1_err"] = top1_err
stats["top5_err"] = top5_err
stats["min_top1_err"] = self.min_top1_err
stats["min_top5_err"] = self.min_top5_err
logging.log_json_stats(stats)
return stats
def finalize_metrics(self, log=True):
"""
Calculate and log the final AVA metrics.
"""
all_preds = torch.cat(self.all_preds, dim=0)
all_ori_boxes = torch.cat(self.all_ori_boxes, dim=0)
all_metadata = torch.cat(self.all_metadata, dim=0)
if self.mode == "test" or (self.full_ava_test and self.mode == "val"):
groundtruth = self.full_groundtruth
else:
groundtruth = self.mini_groundtruth
self.full_map = evaluate_ava(
all_preds,
all_ori_boxes,
all_metadata.tolist(),
self.excluded_keys,
self.class_whitelist,
self.categories,
groundtruth=groundtruth,
video_idx_to_name=self.video_idx_to_name,
)
if log:
stats = {"mode": self.mode, "map": self.full_map}
logging.log_json_stats(stats)
def LogIterStats(self, cur_iter, lr):
"""Log the tracked statistics."""
if (cur_iter % self.LOG_PERIOD == 0
or cur_iter == cfg.SOLVER.MAX_ITER - 1):
stats = self.GetStats(cur_iter, lr)
log_json_stats(stats)
with open(os.path.join(get_output_dir(), 'json_stats.json'),
'a') as f:
f.write(json.dumps(stats, sort_keys=True))
f.write('\n')
def LogIterStats(self, cur_iter, lr, nan=False):
"""Log the tracked statistics."""
if cur_iter % self.LOG_PERIOD == 0 or cur_iter == cfg.SOLVER.MAX_ITER - 1 or nan:
stats = self.GetStats(cur_iter, lr)
log_json_stats(stats)
# X: first dump the statistics
dump_stats = {key.replace('retnet_', 'retnet/'): val for key, val in six.iteritems(stats) \
if isinstance(val, Number)}
self.writer.write_scalars(dump_stats, cur_iter)
if cur_iter % self.LOG_SLOW_PERIOD == 0 and not nan:
self.writer.write_summaries(cur_iter)
def log_iter_stats(self, cur_iter):
"""
Log the stats.
Args:
cur_iter (int): the current iteration of testing.
"""
eta_sec = self.iter_timer.seconds() * (self.overall_iters - cur_iter)
eta = str(datetime.timedelta(seconds=int(eta_sec)))
stats = {
"split": "test_iter",
"cur_iter": "{}".format(cur_iter + 1),
"eta": eta,
"time_diff": self.iter_timer.seconds(),
}
logging.log_json_stats(stats)
def log_epoch_stats(self, cur_epoch):
"""
Log the stats of the current epoch.
Args:
cur_epoch (int): the number of current epoch.
"""
if self.mode in ["val", "test"]:
self.finalize_metrics(log=False)
stats = {
"_type": "{}_epoch".format(self.mode),
"cur_epoch": "{}".format(cur_epoch + 1),
"mode": self.mode,
"map": self.full_map,
"gpu_mem": "{:.2f} GB".format(misc.gpu_mem_usage()),
"RAM": "{:.2f}/{:.2f} GB".format(*misc.cpu_mem_usage()),
}
logging.log_json_stats(stats)
def log_iter_stats(self, cur_epoch, cur_iter):
"""
Log the stats.
Args:
cur_epoch (int): the current epoch.
cur_iter (int): the current iteration.
"""
if (cur_iter + 1) % self.cfg.LOG_PERIOD != 0:
return
eta_sec = self.iter_timer.seconds() * (self.overall_iters - cur_iter)
eta = str(datetime.timedelta(seconds=int(eta_sec)))
if self.mode == "train":
stats = {
"_type": "{}_iter".format(self.mode),
"cur_epoch": "{}".format(cur_epoch + 1),
"cur_iter": "{}".format(cur_iter + 1),
"eta": eta,
"time_diff": self.iter_timer.seconds(),
"mode": self.mode,
"loss": self.loss.get_win_median(),
"lr": self.lr,
}
elif self.mode == "val":
stats = {
"_type": "{}_iter".format(self.mode),
"cur_epoch": "{}".format(cur_epoch + 1),
"cur_iter": "{}".format(cur_iter + 1),
"eta": eta,
"time_diff": self.iter_timer.seconds(),
"mode": self.mode,
}
elif self.mode == "test":
stats = {
"_type": "{}_iter".format(self.mode),
"cur_iter": "{}".format(cur_iter + 1),
"eta": eta,
"time_diff": self.iter_timer.seconds(),
"mode": self.mode,
}
else:
raise NotImplementedError("Unknown mode: {}".format(self.mode))
logging.log_json_stats(stats)
def log_epoch_stats(self, cur_epoch):
"""
Log the stats of the current epoch.
Args:
cur_epoch (int): the number of current epoch.
"""
eta_sec = self.iter_timer.seconds() * (
self.MAX_EPOCH - (cur_epoch + 1) * self.epoch_iters)
eta = str(datetime.timedelta(seconds=int(eta_sec)))
mem_usage = misc.gpu_mem_usage()
stats = {
"_type": "train_epoch",
"epoch": "{}/{}".format(cur_epoch + 1, self._cfg.SOLVER.MAX_EPOCH),
"time_diff": self.iter_timer.seconds(),
"eta": eta,
"mem": int(np.ceil(mem_usage)),
}
infos = {k: v.get_global_avg() for k, v in self.infos}
stats.update(infos)
logging.log_json_stats(stats)
def LogIterStats(self, cur_iter, lr):
"""Log the tracked statistics."""
if (cur_iter % self.LOG_PERIOD == 0
or cur_iter == cfg.SOLVER.MAX_ITER - 1):
eta_seconds = self.iter_timer.average_time * (cfg.SOLVER.MAX_ITER -
cur_iter)
eta = str(datetime.timedelta(seconds=int(eta_seconds)))
mem_stats = c2_py_utils.GetGPUMemoryUsageStats()
mem_usage = np.max(mem_stats['max_by_gpu'][:cfg.NUM_GPUS])
stats = dict(iter=cur_iter,
lr=float(lr),
time=self.iter_timer.average_time,
loss=self.smoothed_total_loss.GetMedianValue(),
eta=eta,
mb_qsize=int(
np.round(
self.smoothed_mb_qsize.GetMedianValue())),
mem=int(np.ceil(mem_usage / 1024 / 1024)))
for k, v in self.smoothed_losses_and_metrics.items():
stats[k] = v.GetMedianValue()
log_json_stats(stats)
def log_average_score(self, cur_epoch):
score_per_vid = {}
for vid in self.score.keys():
psnrs = [x[0] for x in self.score[vid].values()]
ssims = [x[1] for x in self.score[vid].values()]
score_per_vid[vid] = (np.mean(psnrs), np.mean(ssims))
avg_psnr = np.mean([v[0] for k, v in score_per_vid.items()])
avg_ssim = np.mean([v[1] for k, v in score_per_vid.items()])
stats = {
"cur_epoch": cur_epoch,
"_type": "validation_epoch",
"psnr": avg_psnr,
"ssim": avg_ssim
}
logging.log_json_stats(stats)
logging.log_json_stats(score_per_vid)
# with open(Join(self.output_dir, 'videos_scores.csv'), 'w') as f:
# f.write('video_id, psnr, ssim\n')
# for vid in self.score.keys():
# f.write("{},{},{}\n".format(vid, score_per_vid[vid][0], score_per_vid[vid][1]))
return
def log_iter_stats(self, cur_epoch, cur_iter):
"""
log the stats of the current iteration.
Args:
cur_epoch (int): the number of current epoch.
cur_iter (int): the number of current iteration.
"""
if (cur_iter + 1) % self._cfg.LOG_PERIOD != 0:
return
eta_sec = self.iter_timer.seconds() * (self.max_iter - cur_iter - 1)
eta = str(datetime.timedelta(seconds=int(eta_sec)))
stats = {
"_type": "val_iter",
"epoch": "{}/{}".format(cur_epoch + 1, self._cfg.SOLVER.MAX_EPOCH),
"iter": "{}/{}".format(cur_iter + 1, self.max_iter),
"time_diff": self.iter_timer.seconds(),
"eta": eta,
"gpu_mem": "{:.2f} GB".format(misc.gpu_mem_usage()),
}
if not self._cfg.DATA.MULTI_LABEL:
stats["top1_err"] = self.mb_top1_err.get_win_median()
stats["top5_err"] = self.mb_top5_err.get_win_median()
logging.log_json_stats(stats)
def LogIterStats(self, cur_iter, lr):
"""Log the tracked statistics."""
if (cur_iter % self.LOG_PERIOD == 0 or
cur_iter == cfg.SOLVER.MAX_ITER - 1):
eta_seconds = self.iter_timer.average_time * (
cfg.SOLVER.MAX_ITER - cur_iter
)
eta = str(datetime.timedelta(seconds=int(eta_seconds)))
mem_stats = c2_py_utils.GetGPUMemoryUsageStats()
mem_usage = np.max(mem_stats['max_by_gpu'][:cfg.NUM_GPUS])
stats = dict(
iter=cur_iter,
lr=float(lr),
time=self.iter_timer.average_time,
loss=self.smoothed_total_loss.GetMedianValue(),
eta=eta,
mb_qsize=int(
np.round(self.smoothed_mb_qsize.GetMedianValue())
),
mem=int(np.ceil(mem_usage / 1024 / 1024))
)
for k, v in self.smoothed_losses_and_metrics.items():
stats[k] = v.GetMedianValue()
log_json_stats(stats)
def log_iter_stats(self, cur_epoch, cur_iter):
"""
log the stats of the current iteration.
Args:
cur_epoch (int): the number of current epoch.
cur_iter (int): the number of current iteration.
"""
if (cur_iter + 1) % self._cfg.LOG_PERIOD != 0:
return
eta_sec = self.iter_timer.seconds() * (
self.MAX_EPOCH - (cur_epoch * self.epoch_iters + cur_iter + 1))
eta = str(datetime.timedelta(seconds=int(eta_sec)))
mem_usage = misc.gpu_mem_usage()
stats = {
"_type": "train_iter",
"epoch": "{}/{}".format(cur_epoch + 1, self._cfg.SOLVER.MAX_EPOCH),
"iter": "{}/{}".format(cur_iter + 1, self.epoch_iters),
"time_diff": self.iter_timer.seconds(),
"eta": eta,
"mem": int(np.ceil(mem_usage)),
}
infos = {k: v.get_win_avg() for k, v in self.infos}
stats.update(infos)
logging.log_json_stats(stats)
def LogIterStats(self, cur_iter, lr):
"""Log the tracked statistics."""
if (cur_iter % self.log_period == 0 or
cur_iter == self.solver_max_iters - 1):
stats = self.GetStats(cur_iter, lr)
log_json_stats(stats)
def log_epoch_stats(self, cur_epoch):
stats = self.get_epoch_stats(cur_epoch)
lu.log_json_stats(stats)
def log_iter_stats(self, cur_epoch, cur_iter):
if (cur_iter + 1) % cfg.LOG_PERIOD != 0:
return
stats = self.get_iter_stats(cur_epoch, cur_iter)
lu.log_json_stats(stats)
def LogIterStats(self, cur_iter, lr):
"""Log the tracked statistics."""
if (cur_iter % self.LOG_PERIOD == 0 or
cur_iter == cfg.SOLVER.MAX_ITER - 1):
stats = self.GetStats(cur_iter, lr)
log_json_stats(stats)
def LogIterStats(self, cur_iter, lr):
"""Log the tracked statistics."""
if (cur_iter % self.LOG_PERIOD == 0 or
cur_iter == cfg.SOLVER.MAX_ITER - 1):
stats = self.GetStats(cur_iter, lr)
log_json_stats(stats)
def LogIterStats(self, cur_iter, lr):
"""Log the tracked statistics."""
if (cur_iter % self.log_period == 0
or cur_iter == self.solver_max_iters - 1):
stats = self.GetStats(cur_iter, lr)
log_json_stats(stats)
