def __init__(self, cfg):
self._logger = setup_logger(__name__, all_rank=True)
if dist.is_main_process():
self._logger.debug(f'Config File : \n{cfg}')
if cfg.VISUALIZE_DIR and not os.path.isdir(cfg.VISUALIZE_DIR) : os.makedirs(cfg.VISUALIZE_DIR)
self.visualize_dir = cfg.VISUALIZE_DIR
dist.synchronize()
self.test_loader = build_test_loader(cfg)
self.model = build_model(cfg)
self.model.eval()
if dist.is_main_process():
self._logger.debug(f"Model Structure\n{self.model}")
if dist.get_world_size() > 1:
self.model = DistributedDataParallel(self.model, device_ids=[dist.get_local_rank()], broadcast_buffers=False)
self.checkpointer = Checkpointer(
self.model,
cfg.OUTPUT_DIR,
)
self.checkpointer.load(cfg.WEIGHTS)
self.meta_data = MetadataCatalog.get(cfg.LOADER.TEST_DATASET)
self.class_color = [(255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 0), (255, 0, 255)]
def evaluate(self):
if self._distributed:
dist.synchronize()
predictions = dist.gather(self._predictions, dst=0)
predictions = list(itertools.chain(*predictions))
if not dist.is_main_process(): return {}
else:
predictions = self._predictions
if len(predictions) == 0:
self._logger.error(
"[COCOEvaluator] Did not receive valid predictions.")
return {}
if self._output_dir:
file_path = os.path.join(self._output_dir,
"instances_predictions.pth")
with open(file_path, "wb") as f:
torch.save(predictions, f)
self._results = OrderedDict()
if "instances" in predictions[0]: self._eval_instances(predictions)
# Copy so the caller can do whatever with results
return copy.deepcopy(self._results)
def build_hooks(cfg, model, optimizer, scheduler, checkpointer):
ret = list()
ret.append(TrainTimer())
ret.append(LRScheduler(cfg, optimizer, scheduler))
if dist.is_main_process():
ret.append(PeriodicCheckpointer(cfg, checkpointer))
ret.append(EvalHook(cfg))
if dist.is_main_process():
ret.append(IterTimer(cfg))
ret.append(JSONWriter(cfg))
ret.append(WandbWriter(cfg, model))
# ret.append(TensorboardXWriter(cfg))
return ret
def _eval_instances(self):
if self._distributed:
dist.synchronize()
all_predictions = dist.gather(self._pred_instances, dst=0)
if not dist.is_main_process() : return {}
predictions = defaultdict(list)
for predictions_per_rank in all_predictions:
for clsid, lines in predictions_per_rank.items():
predictions[clsid].extend(lines)
del all_predictions
else:
predictions = self._pred_instances
results = OrderedDict()
with tempfile.TemporaryDirectory(prefix="pascal_voc_eval_") as dirname:
res_file_template = os.path.join(dirname, "{}.txt")
aps = defaultdict(float) # iou -> ap per class
for cls_id, cls_name in enumerate(self._category):
pred_cls = predictions.get(cls_id, None)
if pred_cls is None : continue
with open(res_file_template.format(cls_name), "w") as f:
for pred in pred_cls:
line = f"{pred['image_id']} {pred['score']:.3f}"
if 'pred_box' in pred :
xmin, ymin, xmax, ymax = pred['pred_box']
# The inverse of data loading logic in `loader/data/pascal_voc/load_data.py`
xmin += 1
ymin += 1
line = f"{line} {xmin:.1f} {ymin:.1f} {xmax:.1f} {ymax:.1f}"
f.write(f'{line}\n')
thresh = 50
rec, prec, ap = voc_eval(
res_file_template,
self._anno_file_template,
self._image_set_path,
cls_name,
ovthresh=thresh / 100.0,
use_07_metric=self._is_2007,
)
aps[cls_name] = ap * 100
mAP = np.mean(list(aps.values()))
aps['mAP'] = mAP
results["bbox"] = aps
table = create_small_table(results['bbox'])
self._logger.info(f"\n{table}")
return results
def __init__(self, cfg):
super().__init__(cfg)
if cfg.SEED < 0 : cfg.SEED = dist.shared_random_seed()
self._seed = cfg.SEED
seed_all_rng(self._seed)
self._logger.debug(f'Config File : \n{cfg}')
if dist.is_main_process():
if cfg.OUTPUT_DIR and not os.path.isdir(cfg.OUTPUT_DIR) : os.makedirs(cfg.OUTPUT_DIR)
with open(os.path.join(cfg.OUTPUT_DIR, 'config'), 'w') as f:
f.write(cfg.dump())
dist.synchronize()
self.train_loader = build_train_loader(cfg)
self.test_loader = build_test_loader(cfg)
self.train_iter = iter(self.train_loader)
self.model = build_model(cfg)
self.model.train()
if dist.is_main_process():
self._logger.debug(f"Model Structure\n{self.model}")
self.optimizer = build_optimizer(cfg, self.model)
self.optimizer.zero_grad()
self.scheduler = build_lr_scheduler(cfg, self.optimizer)
self.accumulate = cfg.SOLVER.ACCUMULATE
if dist.get_world_size() > 1:
self.model = DistributedDataParallel(self.model, device_ids=[dist.get_local_rank()], broadcast_buffers=False)
self.weight_path = cfg.WEIGHTS
self.checkpointer = Checkpointer(
self.model,
cfg.OUTPUT_DIR,
optimizer=self.optimizer,
scheduler=self.scheduler,
)
self.evaluator = build_evaluator(cfg)
hooks = build_hooks(cfg, self.model, self.optimizer, self.scheduler, self.checkpointer)
self.register_hooks(hooks)
def _write_metrics(self, metrics_dict: dict):
metrics_dict = {
k: v.detach().cpu().item() if isinstance(v, torch.Tensor) else float(v)
for k, v in metrics_dict.items()
}
all_metrics_dict = dist.gather(metrics_dict)
if dist.is_main_process():
if "data_time" in all_metrics_dict[0]:
data_time = np.max([x.pop("data_time") for x in all_metrics_dict])
self.storage.put_scalar("data_time", data_time)
# average the rest metrics
metrics_dict = {
k: np.mean([x[k] for x in all_metrics_dict]) for k in all_metrics_dict[0].keys()
}
total_losses_reduced = sum(loss for loss in metrics_dict.values())
self.storage.put_scalar("total_loss", total_losses_reduced)
if len(metrics_dict) > 1:
self.storage.put_scalars(**metrics_dict)
def main(args):
cfg = get_cfg(args.config_file)
model = build_model(cfg)
train_loader = build_train_loader(cfg)
test_loader = build_test_loader(cfg)
if dist.is_main_process():
logger.info(f'Model Structure\n{model}')
logger.info(f'Backbone Network\n{model.backbone}')
logger.debug(f'Backbone Output Shape : {model.backbone.output_shape()}')
logger.debug(f'Backbone Output Features : {model.backbone.out_features}')
logger.debug(f'Backbone Stride : {model.backbone.out_feature_strides}')
logger.debug(f'Backbone Output Channels : {model.backbone.out_feature_channels}')
train_iter = iter(train_loader)
input_data = next(train_iter)
logger.debug(f'Input Data Structure\n{input_data[0]}')
total_param = sum(p.numel() for p in model.parameters())
logger.debug(f'The Number of Model Parameters : {total_param}')
def _eval_proposals(self):
if self._distributed:
dist.synchronize()
all_predictions = dist.gather(self._pred_proposals, dst=0)
if not dist.is_main_process(): return {}
predictions = list()
for predictions_per_rank in all_predictions:
predictions.extend(predictions_per_rank)
del all_predictions
else:
predictions = self._pred_proposals
results = OrderedDict()
mAP = defaultdict(float) # iou -> ap
for thresh in range(50, 100, 5):
rec, prec, ap = voc_eval(
predictions,
self._anno_file_template,
self._image_set_path,
ovthresh=thresh / 100.0,
use_07_metric=self._is_2007,
)
mAP[thresh] = ap * 100
results["proposal"] = {
"AP": np.mean(list(mAP.values())),
"AP50": mAP[50],
"AP75": mAP[75]
}
table = create_small_table(results['proposal'])
self._logger.info(f"\n{table}")
return results