def log_iter_stats(self, cur_epoch, cur_iter, global_iter): """ log the stats of the current iteration. Args: cur_epoch (int): the number of current epoch. cur_iter (int): the number of current iteration in current epoch. global_iter (int): the number of current iteration in training phase """ if (cur_iter + 1) % self._cfg.LOG_PERIOD != 0: return eta_sec = self.iter_timer.seconds() * ( self.MAX_ITERS - (cur_epoch * self.epoch_iters + cur_iter + 1)) eta = str(datetime.timedelta(seconds=int(eta_sec))) stats = { "_type": "train_iter", "global_iter": "{}/{}".format(global_iter, self.MAX_ITERS), "epoch": "{}/{}".format(cur_epoch + 1, self.MAX_EPOCH), "iter": "{}/{}".format(cur_iter + 1, self.epoch_iters), "time_diff": self.iter_timer.seconds(), "eta": eta, "loss": self.loss.get_win_median(), "lr": self.lr, "gpu_mem": int(np.ceil(misc.gpu_mem_usage())), } if not self._cfg.DATA.MULTI_LABEL: stats["top1_err"] = self.mb_top1_err.get_win_median() stats["top5_err"] = self.mb_top5_err.get_win_median() logging.log_json_stats(stats)
def log_epoch_stats(self, cur_epoch): """ Log the stats of the current epoch. Args: cur_epoch (int): the number of current epoch. """ eta_sec = self.iter_timer.seconds() * ( self.MAX_ITERS - (cur_epoch + 1) * self.epoch_iters) eta = str(datetime.timedelta(seconds=int(eta_sec))) stats = { "_type": "train_epoch", "epoch": "{}/{}".format(cur_epoch + 1, self.MAX_EPOCH), "time_diff": self.iter_timer.seconds(), "eta": eta, "lr": self.lr, "gpu_mem": int(np.ceil(misc.gpu_mem_usage())), } if not self._cfg.DATA.MULTI_LABEL: top1_err = self.num_top1_mis / self.num_samples top5_err = self.num_top5_mis / self.num_samples avg_loss = self.loss_total / self.num_samples stats["top1_err"] = top1_err stats["top5_err"] = top5_err stats["loss"] = avg_loss logging.log_json_stats(stats)
def finalize_metrics(self, ks=(1, 5)): """ Calculate and log the final ensembled metrics. ks (tuple): list of top-k values for topk_accuracies. For example, ks = (1, 5) correspods to top-1 and top-5 accuracy. """ if not all(self.clip_count == self.num_clips): logger.warning("clip count {} ~= num clips {}".format( ", ".join([ "{}: {}".format(i, k) for i, k in enumerate(self.clip_count.tolist()) ]), self.num_clips, )) stats = {"split": "test_final"} if self.multi_label: map = get_map(self.video_preds.cpu().numpy(), self.video_labels.cpu().numpy()) stats["map"] = map else: num_topks_correct = metrics.topks_correct(self.video_preds, self.video_labels, ks) topks = [(x / self.video_preds.size(0)) * 100.0 for x in num_topks_correct] assert len({len(ks), len(topks)}) == 1 for k, topk in zip(ks, topks): stats["top{}_acc".format(k)] = "{:.{prec}f}".format(topk, prec=2) logging.log_json_stats(stats)
def log_epoch_stats(self, cur_epoch): """ Log the stats of the current epoch. Args: cur_epoch (int): the number of current epoch. """ mem_usage = misc.gpu_mem_usage() stats = { "_type": "val_epoch", "epoch": "{}/{}".format(cur_epoch + 1, self.MAX_EPOCH), "time_diff": self.iter_timer.seconds(), "gpu_mem": int(np.ceil(mem_usage)), } if self._cfg.DATA.MULTI_LABEL: stats["map"] = get_map( torch.cat(self.all_preds).cpu().numpy(), torch.cat(self.all_labels).cpu().numpy(), ) self.max_map = max(self.max_map, stats["map"]) stats["max_map"] = self.max_map else: top1_err = self.num_top1_mis / self.num_samples top5_err = self.num_top5_mis / self.num_samples self.min_top1_err = min(self.min_top1_err, top1_err) self.min_top5_err = min(self.min_top5_err, top5_err) stats["top1_err"] = top1_err stats["top5_err"] = top5_err stats["min_top1_err"] = self.min_top1_err stats["min_top5_err"] = self.min_top5_err logging.log_json_stats(stats) return stats
def finalize_metrics(self, log=True): """ Calculate and log the final AVA metrics. """ all_preds = torch.cat(self.all_preds, dim=0) all_ori_boxes = torch.cat(self.all_ori_boxes, dim=0) all_metadata = torch.cat(self.all_metadata, dim=0) if self.mode == "test" or (self.full_ava_test and self.mode == "val"): groundtruth = self.full_groundtruth else: groundtruth = self.mini_groundtruth self.full_map = evaluate_ava( all_preds, all_ori_boxes, all_metadata.tolist(), self.excluded_keys, self.class_whitelist, self.categories, groundtruth=groundtruth, video_idx_to_name=self.video_idx_to_name, ) if log: stats = {"mode": self.mode, "map": self.full_map} logging.log_json_stats(stats)
def LogIterStats(self, cur_iter, lr): """Log the tracked statistics.""" if (cur_iter % self.LOG_PERIOD == 0 or cur_iter == cfg.SOLVER.MAX_ITER - 1): stats = self.GetStats(cur_iter, lr) log_json_stats(stats) with open(os.path.join(get_output_dir(), 'json_stats.json'), 'a') as f: f.write(json.dumps(stats, sort_keys=True)) f.write('\n')
def LogIterStats(self, cur_iter, lr, nan=False): """Log the tracked statistics.""" if cur_iter % self.LOG_PERIOD == 0 or cur_iter == cfg.SOLVER.MAX_ITER - 1 or nan: stats = self.GetStats(cur_iter, lr) log_json_stats(stats) # X: first dump the statistics dump_stats = {key.replace('retnet_', 'retnet/'): val for key, val in six.iteritems(stats) \ if isinstance(val, Number)} self.writer.write_scalars(dump_stats, cur_iter) if cur_iter % self.LOG_SLOW_PERIOD == 0 and not nan: self.writer.write_summaries(cur_iter)
def log_iter_stats(self, cur_iter): """ Log the stats. Args: cur_iter (int): the current iteration of testing. """ eta_sec = self.iter_timer.seconds() * (self.overall_iters - cur_iter) eta = str(datetime.timedelta(seconds=int(eta_sec))) stats = { "split": "test_iter", "cur_iter": "{}".format(cur_iter + 1), "eta": eta, "time_diff": self.iter_timer.seconds(), } logging.log_json_stats(stats)
def log_epoch_stats(self, cur_epoch): """ Log the stats of the current epoch. Args: cur_epoch (int): the number of current epoch. """ if self.mode in ["val", "test"]: self.finalize_metrics(log=False) stats = { "_type": "{}_epoch".format(self.mode), "cur_epoch": "{}".format(cur_epoch + 1), "mode": self.mode, "map": self.full_map, "gpu_mem": "{:.2f} GB".format(misc.gpu_mem_usage()), "RAM": "{:.2f}/{:.2f} GB".format(*misc.cpu_mem_usage()), } logging.log_json_stats(stats)
def log_iter_stats(self, cur_epoch, cur_iter): """ Log the stats. Args: cur_epoch (int): the current epoch. cur_iter (int): the current iteration. """ if (cur_iter + 1) % self.cfg.LOG_PERIOD != 0: return eta_sec = self.iter_timer.seconds() * (self.overall_iters - cur_iter) eta = str(datetime.timedelta(seconds=int(eta_sec))) if self.mode == "train": stats = { "_type": "{}_iter".format(self.mode), "cur_epoch": "{}".format(cur_epoch + 1), "cur_iter": "{}".format(cur_iter + 1), "eta": eta, "time_diff": self.iter_timer.seconds(), "mode": self.mode, "loss": self.loss.get_win_median(), "lr": self.lr, } elif self.mode == "val": stats = { "_type": "{}_iter".format(self.mode), "cur_epoch": "{}".format(cur_epoch + 1), "cur_iter": "{}".format(cur_iter + 1), "eta": eta, "time_diff": self.iter_timer.seconds(), "mode": self.mode, } elif self.mode == "test": stats = { "_type": "{}_iter".format(self.mode), "cur_iter": "{}".format(cur_iter + 1), "eta": eta, "time_diff": self.iter_timer.seconds(), "mode": self.mode, } else: raise NotImplementedError("Unknown mode: {}".format(self.mode)) logging.log_json_stats(stats)
def log_epoch_stats(self, cur_epoch): """ Log the stats of the current epoch. Args: cur_epoch (int): the number of current epoch. """ eta_sec = self.iter_timer.seconds() * ( self.MAX_EPOCH - (cur_epoch + 1) * self.epoch_iters) eta = str(datetime.timedelta(seconds=int(eta_sec))) mem_usage = misc.gpu_mem_usage() stats = { "_type": "train_epoch", "epoch": "{}/{}".format(cur_epoch + 1, self._cfg.SOLVER.MAX_EPOCH), "time_diff": self.iter_timer.seconds(), "eta": eta, "mem": int(np.ceil(mem_usage)), } infos = {k: v.get_global_avg() for k, v in self.infos} stats.update(infos) logging.log_json_stats(stats)
def LogIterStats(self, cur_iter, lr): """Log the tracked statistics.""" if (cur_iter % self.LOG_PERIOD == 0 or cur_iter == cfg.SOLVER.MAX_ITER - 1): eta_seconds = self.iter_timer.average_time * (cfg.SOLVER.MAX_ITER - cur_iter) eta = str(datetime.timedelta(seconds=int(eta_seconds))) mem_stats = c2_py_utils.GetGPUMemoryUsageStats() mem_usage = np.max(mem_stats['max_by_gpu'][:cfg.NUM_GPUS]) stats = dict(iter=cur_iter, lr=float(lr), time=self.iter_timer.average_time, loss=self.smoothed_total_loss.GetMedianValue(), eta=eta, mb_qsize=int( np.round( self.smoothed_mb_qsize.GetMedianValue())), mem=int(np.ceil(mem_usage / 1024 / 1024))) for k, v in self.smoothed_losses_and_metrics.items(): stats[k] = v.GetMedianValue() log_json_stats(stats)
def log_average_score(self, cur_epoch): score_per_vid = {} for vid in self.score.keys(): psnrs = [x[0] for x in self.score[vid].values()] ssims = [x[1] for x in self.score[vid].values()] score_per_vid[vid] = (np.mean(psnrs), np.mean(ssims)) avg_psnr = np.mean([v[0] for k, v in score_per_vid.items()]) avg_ssim = np.mean([v[1] for k, v in score_per_vid.items()]) stats = { "cur_epoch": cur_epoch, "_type": "validation_epoch", "psnr": avg_psnr, "ssim": avg_ssim } logging.log_json_stats(stats) logging.log_json_stats(score_per_vid) # with open(Join(self.output_dir, 'videos_scores.csv'), 'w') as f: # f.write('video_id, psnr, ssim\n') # for vid in self.score.keys(): # f.write("{},{},{}\n".format(vid, score_per_vid[vid][0], score_per_vid[vid][1])) return
def log_iter_stats(self, cur_epoch, cur_iter): """ log the stats of the current iteration. Args: cur_epoch (int): the number of current epoch. cur_iter (int): the number of current iteration. """ if (cur_iter + 1) % self._cfg.LOG_PERIOD != 0: return eta_sec = self.iter_timer.seconds() * (self.max_iter - cur_iter - 1) eta = str(datetime.timedelta(seconds=int(eta_sec))) stats = { "_type": "val_iter", "epoch": "{}/{}".format(cur_epoch + 1, self._cfg.SOLVER.MAX_EPOCH), "iter": "{}/{}".format(cur_iter + 1, self.max_iter), "time_diff": self.iter_timer.seconds(), "eta": eta, "gpu_mem": "{:.2f} GB".format(misc.gpu_mem_usage()), } if not self._cfg.DATA.MULTI_LABEL: stats["top1_err"] = self.mb_top1_err.get_win_median() stats["top5_err"] = self.mb_top5_err.get_win_median() logging.log_json_stats(stats)
def LogIterStats(self, cur_iter, lr): """Log the tracked statistics.""" if (cur_iter % self.LOG_PERIOD == 0 or cur_iter == cfg.SOLVER.MAX_ITER - 1): eta_seconds = self.iter_timer.average_time * ( cfg.SOLVER.MAX_ITER - cur_iter ) eta = str(datetime.timedelta(seconds=int(eta_seconds))) mem_stats = c2_py_utils.GetGPUMemoryUsageStats() mem_usage = np.max(mem_stats['max_by_gpu'][:cfg.NUM_GPUS]) stats = dict( iter=cur_iter, lr=float(lr), time=self.iter_timer.average_time, loss=self.smoothed_total_loss.GetMedianValue(), eta=eta, mb_qsize=int( np.round(self.smoothed_mb_qsize.GetMedianValue()) ), mem=int(np.ceil(mem_usage / 1024 / 1024)) ) for k, v in self.smoothed_losses_and_metrics.items(): stats[k] = v.GetMedianValue() log_json_stats(stats)
def log_iter_stats(self, cur_epoch, cur_iter): """ log the stats of the current iteration. Args: cur_epoch (int): the number of current epoch. cur_iter (int): the number of current iteration. """ if (cur_iter + 1) % self._cfg.LOG_PERIOD != 0: return eta_sec = self.iter_timer.seconds() * ( self.MAX_EPOCH - (cur_epoch * self.epoch_iters + cur_iter + 1)) eta = str(datetime.timedelta(seconds=int(eta_sec))) mem_usage = misc.gpu_mem_usage() stats = { "_type": "train_iter", "epoch": "{}/{}".format(cur_epoch + 1, self._cfg.SOLVER.MAX_EPOCH), "iter": "{}/{}".format(cur_iter + 1, self.epoch_iters), "time_diff": self.iter_timer.seconds(), "eta": eta, "mem": int(np.ceil(mem_usage)), } infos = {k: v.get_win_avg() for k, v in self.infos} stats.update(infos) logging.log_json_stats(stats)
def LogIterStats(self, cur_iter, lr): """Log the tracked statistics.""" if (cur_iter % self.log_period == 0 or cur_iter == self.solver_max_iters - 1): stats = self.GetStats(cur_iter, lr) log_json_stats(stats)
def log_epoch_stats(self, cur_epoch): stats = self.get_epoch_stats(cur_epoch) lu.log_json_stats(stats)
def log_iter_stats(self, cur_epoch, cur_iter): if (cur_iter + 1) % cfg.LOG_PERIOD != 0: return stats = self.get_iter_stats(cur_epoch, cur_iter) lu.log_json_stats(stats)
def LogIterStats(self, cur_iter, lr): """Log the tracked statistics.""" if (cur_iter % self.LOG_PERIOD == 0 or cur_iter == cfg.SOLVER.MAX_ITER - 1): stats = self.GetStats(cur_iter, lr) log_json_stats(stats)