class TrainingStats(object): """Track vital training statistics.""" def __init__(self, model): # Window size for smoothing tracked values (with median filtering) self.WIN_SZ = int(20 / cfg.NUM_GPUS) # Output logging period in SGD iterations self.LOG_PERIOD = int(20 /cfg.NUM_GPUS) self.smoothed_losses_and_metrics = { key: SmoothedValue(self.WIN_SZ) for key in model.losses + model.metrics } self.losses_and_metrics = { key: 0 for key in model.losses + model.metrics } self.smoothed_total_loss = SmoothedValue(self.WIN_SZ) self.smoothed_mb_qsize = SmoothedValue(self.WIN_SZ) self.iter_total_loss = np.nan self.iter_timer = Timer() self.model = model def IterTic(self): self.iter_timer.tic() def IterToc(self): return self.iter_timer.toc(average=False) def ResetIterTimer(self): self.iter_timer.reset() def UpdateIterStats(self): """Update tracked iteration statistics.""" for k in self.losses_and_metrics.keys(): if k in self.model.losses: self.losses_and_metrics[k] = nu.sum_multi_gpu_blob(k) else: self.losses_and_metrics[k] = nu.average_multi_gpu_blob(k) for k, v in self.smoothed_losses_and_metrics.items(): v.AddValue(self.losses_and_metrics[k]) self.iter_total_loss = np.sum( np.array([self.losses_and_metrics[k] for k in self.model.losses]) ) self.smoothed_total_loss.AddValue(self.iter_total_loss) self.smoothed_mb_qsize.AddValue( self.model.roi_data_loader._minibatch_queue.qsize() ) def LogIterStats(self, cur_iter, lr): """Log the tracked statistics.""" num_iter_per_epoch = self.model.roi_data_loader.get_num_iter_per_epoch() if (cur_iter % self.LOG_PERIOD == 0 or cur_iter == cfg.SOLVER.MAX_ITER * num_iter_per_epoch - 1): stats = self.GetStats(cur_iter, lr) log_json_stats(stats) def GetStats(self, cur_iter, lr): num_iter_per_epoch = self.model.roi_data_loader.get_num_iter_per_epoch() eta_seconds = self.iter_timer.average_time * ( cfg.SOLVER.MAX_ITER * num_iter_per_epoch - cur_iter ) eta = str(datetime.timedelta(seconds=int(eta_seconds))) mem_stats = c2_py_utils.GetGPUMemoryUsageStats() mem_usage = np.max(mem_stats['max_by_gpu'][:cfg.NUM_GPUS]) stats = dict( iter=cur_iter, lr=float(lr), time=self.iter_timer.average_time, loss=self.smoothed_total_loss.GetAverageValue(), eta=eta, mb_qsize=int( np.round(self.smoothed_mb_qsize.GetAverageValue()) ), mem=int(np.ceil(mem_usage / 1024 / 1024)) ) for k, v in self.smoothed_losses_and_metrics.items(): stats[k] = v.GetAverageValue() return stats
class TrainingStats(object): """Track vital training statistics.""" def __init__(self, model): # Window size for smoothing tracked values (with median filtering) self.WIN_SZ = int(1280 / cfg.NUM_GPUS) # Output logging period in SGD iterations self.LOG_PERIOD = int(1280 / cfg.NUM_GPUS) self.smoothed_losses_and_metrics = { key: SmoothedValue(self.WIN_SZ) for key in model.losses + model.metrics } self.losses_and_metrics = { key: 0 for key in model.losses + model.metrics } self.smoothed_total_loss = SmoothedValue(self.WIN_SZ) self.smoothed_mb_qsize = SmoothedValue(self.WIN_SZ) self.iter_total_loss = np.nan self.iter_timer = Timer() self.model = model self.mem = dict() self.mem = None def IterTic(self): self.iter_timer.tic() def IterToc(self): return self.iter_timer.toc(average=False) def ResetIterTimer(self): self.iter_timer.reset() def UpdateIterStats(self): """Update tracked iteration statistics.""" for k in self.losses_and_metrics.keys(): self.losses_and_metrics[k] = nu.average_multi_gpu_blob(k) for k, v in self.smoothed_losses_and_metrics.items(): v.AddValue(self.losses_and_metrics[k]) self.iter_total_loss = np.sum( np.array([self.losses_and_metrics[k] for k in self.model.losses])) self.smoothed_total_loss.AddValue(self.iter_total_loss) self.smoothed_mb_qsize.AddValue( self.model.roi_data_loader._minibatch_queue.qsize()) if self.mem is not None: self.GetMem() def LogIterStats(self, cur_iter, lr): """Log the tracked statistics.""" if (cur_iter % self.LOG_PERIOD == 0 or cur_iter == cfg.SOLVER.MAX_ITER - 1): stats = self.GetStats(cur_iter, lr) log_json_stats(stats) if self.mem is not None: mem_sorted = sorted(self.mem.items(), key=lambda d: d[1]) print(mem_sorted) def GetStats(self, cur_iter, lr): eta_seconds = self.iter_timer.average_time * (cfg.SOLVER.MAX_ITER - cur_iter) eta = str(datetime.timedelta(seconds=int(eta_seconds))) mem_stats = c2_py_utils.GetGPUMemoryUsageStats() mem_usage = np.max(mem_stats['max_by_gpu'][:cfg.NUM_GPUS]) stats = dict(iter=cur_iter, lr=float(lr), time=self.iter_timer.average_time, loss=self.smoothed_total_loss.GetAverageValue(), eta=eta, mb_qsize=int( np.round(self.smoothed_mb_qsize.GetAverageValue())), mem=int(np.ceil(mem_usage / 1024 / 1024))) for k, v in self.smoothed_losses_and_metrics.items(): stats[k] = v.GetAverageValue() return stats def is_grad(self, b): name = str(b) return name.endswith("_grad") def is_shared(self, b): name = str(b) return name.endswith("_shared") def GetMem(self): for op_idx in range(len(self.model.net._net.op)): op = self.model.net._net.op[op_idx] for b in list(op.output): if self.is_grad(b): pass elif self.is_shared(b): pass else: continue blob = workspace.FetchBlob(str(b)) if b not in self.mem.keys(): self.mem[str(b)] = 0 self.mem[str(b)] = max(self.mem[str(b)], blob.size)
class TrainingStats(object): """Track vital training statistics.""" def __init__(self, model, tensorflow_board=None): # Window size for smoothing tracked values (with median filtering) self.WIN_SZ = 20 # Output logging period in SGD iterations self.LOG_PERIOD = 20 self.smoothed_losses_and_metrics = { key: SmoothedValue(self.WIN_SZ) for key in model.losses + model.metrics } self.losses_and_metrics = { key: 0 for key in model.losses + model.metrics } self.smoothed_total_loss = SmoothedValue(self.WIN_SZ) self.smoothed_mb_qsize = SmoothedValue(self.WIN_SZ) self.iter_total_loss = np.nan self.iter_timer = Timer() self.model = model self.tblogger = tensorflow_board self.tb_ignored_keys = ['iter', 'eta', 'mb_qsize', 'mem', 'time'] def IterTic(self): self.iter_timer.tic() def IterToc(self): return self.iter_timer.toc(average=False) def ResetIterTimer(self): self.iter_timer.reset() def UpdateIterStats(self): """Update tracked iteration statistics.""" for k in self.losses_and_metrics.keys(): if k in self.model.losses: self.losses_and_metrics[k] = nu.sum_multi_gpu_blob(k) else: self.losses_and_metrics[k] = nu.average_multi_gpu_blob(k) for k, v in self.smoothed_losses_and_metrics.items(): v.AddValue(self.losses_and_metrics[k]) self.iter_total_loss = np.sum( np.array([self.losses_and_metrics[k] for k in self.model.losses])) self.smoothed_total_loss.AddValue(self.iter_total_loss) self.smoothed_mb_qsize.AddValue( self.model.roi_data_loader._minibatch_queue.qsize()) def LogIterStats(self, cur_iter, lr): """Log the tracked statistics.""" if (cur_iter % self.LOG_PERIOD == 0 or cur_iter == cfg.SOLVER.MAX_ITER - 1): stats = self.GetStats(cur_iter, lr) log_json_stats(stats) if self.tblogger: self.tb_log_stats(stats, cur_iter) def tb_log_stats(self, stats, cur_iter): """Log the tracked statistics to tensorboard""" for k in stats: if k not in self.tb_ignored_keys: v = stats[k] if isinstance(v, dict): self.tb_log_stats(v, cur_iter) else: self.tblogger.write_scalars({k: v}, cur_iter) def GetStats(self, cur_iter, lr): eta_seconds = self.iter_timer.average_time * (cfg.SOLVER.MAX_ITER - cur_iter) eta = str(datetime.timedelta(seconds=int(eta_seconds))) mem_stats = c2_py_utils.GetGPUMemoryUsageStats() mem_usage = np.max(mem_stats['max_by_gpu'][:cfg.NUM_GPUS]) stats = dict(iter=cur_iter, lr=float(lr), time=self.iter_timer.average_time, loss=self.smoothed_total_loss.GetMedianValue(), eta=eta, mb_qsize=int( np.round(self.smoothed_mb_qsize.GetMedianValue())), mem=int(np.ceil(mem_usage / 1024 / 1024))) for k, v in self.smoothed_losses_and_metrics.items(): stats[k] = v.GetMedianValue() return stats
class TrainingStats(object): """Track vital training statistics.""" def __init__(self, model): # Window size for smoothing tracked values (with median filtering) self.WIN_SZ = 1 # Output logging period in SGD iterations self.LOG_PERIOD = 1 self.smoothed_losses_and_metrics = { key: SmoothedValue(self.WIN_SZ) for key in model.losses + model.metrics } self.losses_and_metrics = { key: 0 for key in model.losses + model.metrics } self.smoothed_total_loss = SmoothedValue(self.WIN_SZ) self.smoothed_mb_qsize = SmoothedValue(self.WIN_SZ) self.iter_total_loss = np.nan self.iter_timer = Timer() self.model = model def IterTic(self): self.iter_timer.tic() def IterToc(self): return self.iter_timer.toc(average=False) def ResetIterTimer(self): self.iter_timer.reset() def UpdateIterStats(self): """Update tracked iteration statistics.""" for k in self.losses_and_metrics.keys(): if k in self.model.losses: self.losses_and_metrics[k] = nu.sum_multi_gpu_blob(k) else: self.losses_and_metrics[k] = nu.average_multi_gpu_blob(k) for k, v in self.smoothed_losses_and_metrics.items(): v.AddValue(self.losses_and_metrics[k]) self.iter_total_loss = np.sum( np.array([self.losses_and_metrics[k] for k in self.model.losses])) self.smoothed_total_loss.AddValue(self.iter_total_loss) self.smoothed_mb_qsize.AddValue( self.model.roi_data_loader._minibatch_queue.qsize()) def LogIterStats(self, cur_iter, lr): """Log the tracked statistics.""" if (cur_iter % self.LOG_PERIOD == 0 or cur_iter == cfg.SOLVER.MAX_ITER - 1): stats = self.GetStats(cur_iter, lr) log_json_stats(stats) def GetStats(self, cur_iter, lr): eta_seconds = self.iter_timer.average_time * (cfg.SOLVER.MAX_ITER - cur_iter) eta = str(datetime.timedelta(seconds=int(eta_seconds))) mem_stats = c2_py_utils.GetGPUMemoryUsageStats() mem_usage = np.max(mem_stats['max_by_gpu'][:cfg.NUM_GPUS]) stats = dict( iter=cur_iter, lr=float(lr), time=self.iter_timer.average_time, loss=self.smoothed_total_loss.GetMedianValue(), eta=eta, mb_qsize=int(np.round(self.smoothed_mb_qsize.GetMedianValue())), mem=int(np.ceil(mem_usage / 1024 / 1024)), ) if cfg.TRAIN.DA_FADE_IN: stats['da_weight'] = self.model.da_fade_in.get_weight() if cfg.TRAIN.PADA: stats[ 'avg_pada_weight'] = self.model.class_weight_db.get_avg_pada_weight( ) stats[ 'total_detects'] = self.model.class_weight_db.total_sum_softmax.sum( ) / 2 stats['KL_div'] = self.model.class_weight_db.get_KL_to_init() stats['accuracy_fg'] = self.model.class_weight_db.fg_acc.get() stats[ 'acc_fg_weighted'] = self.model.class_weight_db.weighted_fg_acc.get( ) target_dist = self.model.class_weight_db.get_dist() print('target_dist: {}'.format(list(target_dist))) class_weights = self.model.class_weight_db.class_weights print('class_weights: {}'.format(list(class_weights))) classes = np.array( dummy_datasets.get_coco_dataset().classes.values(), dtype=str) for dist in [target_dist, class_weights]: order = np.argsort(dist)[::-1] o_target_dist = target_dist[order] o_classes = classes[order] cwo = class_weights[order] print("dist tops: ", end='') for prob, w, c in list(zip(o_target_dist, cwo, o_classes))[:5]: print("{}:{:.3f} ({:.3f})".format(c, prob, w), end='; ') print() print() for k, v in self.smoothed_losses_and_metrics.items(): stats[k] = v.GetMedianValue() # for k,v in stats.items(): # print(k,v) return stats
class TrainingStats(object): """Track vital training statistics.""" def __init__(self, model, writer): # Window size for smoothing tracked values (with median filtering) self.WIN_SZ = 20 # Output logging period in SGD iterations self.LOG_PERIOD = 20 self.smoothed_losses_and_metrics = { key: SmoothedValue(self.WIN_SZ) for key in model.losses + model.metrics } self.losses_and_metrics = { key: 0 for key in model.losses + model.metrics } self.smoothed_total_loss = SmoothedValue(self.WIN_SZ) self.smoothed_mb_qsize = SmoothedValue(self.WIN_SZ) self.iter_total_loss = np.nan self.iter_timer = Timer() self.model = model self.writer = writer def IterTic(self): self.iter_timer.tic() def IterToc(self): return self.iter_timer.toc(average=False) def ResetIterTimer(self): self.iter_timer.reset() def UpdateIterStats(self, i_iter=0): """Update tracked iteration statistics.""" for k in self.losses_and_metrics.keys(): if k in self.model.losses: self.losses_and_metrics[k] = nu.sum_multi_gpu_blob(k) else: self.losses_and_metrics[k] = nu.average_multi_gpu_blob(k) for k, v in self.smoothed_losses_and_metrics.items(): v.AddValue(self.losses_and_metrics[k]) self.writer.add_scalar(k, self.losses_and_metrics[k], i_iter) self.iter_total_loss = np.sum( np.array([self.losses_and_metrics[k] for k in self.model.losses]) ) self.smoothed_total_loss.AddValue(self.iter_total_loss) # self.smoothed_mb_qsize.AddValue( # self.model.roi_data_loader._minibatch_queue.qsize() # ) def LogIterStats(self, cur_iter, lr): """Log the tracked statistics.""" if (cur_iter % self.LOG_PERIOD == 0 or cur_iter == cfg.SOLVER.MAX_ITER - 1): stats = self.GetStats(cur_iter, lr) log_json_stats(stats) def GetStats(self, cur_iter, lr): eta_seconds = self.iter_timer.average_time * ( cfg.SOLVER.MAX_ITER - cur_iter ) eta = str(datetime.timedelta(seconds=int(eta_seconds))) mem_stats = c2_py_utils.GetGPUMemoryUsageStats() mem_usage = np.max(mem_stats['max_by_gpu'][:cfg.NUM_GPUS]) stats = dict( iter=cur_iter, lr=float(lr), time=self.iter_timer.average_time, loss=self.smoothed_total_loss.GetMedianValue(), eta=eta, # mb_qsize=int( # np.round(self.smoothed_mb_qsize.GetMedianValue()) # ), mem=int(np.ceil(mem_usage / 1024 / 1024)) ) for k, v in self.smoothed_losses_and_metrics.items(): stats[k] = v.GetMedianValue() return stats def SaveTrainingStates(self, save_file): save_state = { key:0 for key in self.smoothed_losses_and_metrics.keys() } save_smooth_state = { key:0 for key in self.smoothed_losses_and_metrics.keys() } for k, v in self.smoothed_losses_and_metrics.items(): save_state[k] = v.series save_smooth_state[k] = v.smooth_series save_state['total_loss'] = self.smoothed_total_loss.series save_smooth_state['total_loss'] = self.smoothed_total_loss.series # print(save_state) with open(save_file, 'w') as f: json.dump(save_state, f) # save smooth state save_smooth_file = save_file.replace('.json', '_smooth.json') with open(save_smooth_file, 'w') as f: json.dump(save_smooth_state, f)