def visualize_training(self, batched_inputs, results):
#from herbarium.utils.visualizer import Visualizer
assert len(batched_inputs) == len(results), "Cannot visualize inputs and results of different sizes"
storage = get_event_storage()
y_true = batched_inputs["annotations"]
#v_gt = Visualizer(gt_annotation, results)
vis_name = f"per-batch f1 score"
storage.put_scalar(vis_name, f1_score(y_true, results))
def _write_metrics(
self,
loss_dict: Dict[str, torch.Tensor],
data_time: float,
controller_time: float,
base_time: float,
prefix: str = "",
):
"""
Args:
loss_dict (dict): dict of scalar losses
data_time (float): time taken by the dataloader iteration
"""
metrics_dict = {
k: v.detach().cpu().item()
for k, v in loss_dict.items()
}
metrics_dict["data_time"] = data_time
metrics_dict["controller_time"] = controller_time
metrics_dict["base_time"] = base_time
# Gather metrics among all workers for logging
# This assumes we do DDP-style training, which is currently the only
# supported method in herbarium.
all_metrics_dict = comm.gather(metrics_dict)
if comm.is_main_process():
storage = get_event_storage()
# data_time among workers can have high variance. The actual latency
# caused by data_time is the maximum among workers.
data_time = np.max([x.pop("data_time") for x in all_metrics_dict])
controller_time = np.max(
[x.pop("controller_time") for x in all_metrics_dict])
base_time = np.max([x.pop("base_time") for x in all_metrics_dict])
storage.put_scalar("data_time", data_time)
storage.put_scalar("controller_time", controller_time)
storage.put_scalar("base_time", base_time)
# average the rest metrics
metrics_dict = {
k: np.mean([x[k] for x in all_metrics_dict])
for k in all_metrics_dict[0].keys()
}
total_losses_reduced = sum(metrics_dict.values())
if not np.isfinite(total_losses_reduced):
raise FloatingPointError(
f"Loss became infinite or NaN at iteration={self.iter}!\n"
f"loss_dict = {metrics_dict}")
storage.put_scalar("{}total_loss".format(prefix),
total_losses_reduced)
if len(metrics_dict) > 0:
storage.put_scalars(**metrics_dict)
def forward(self, batched_inputs: Tuple[Dict[str, Tensor]]):
"""
Args:
batched_inputs: a list, batched outputs of :class:`DatasetMapper` .
Each item in the list contains the inputs for one image.
For now, each item in the list is a dict that contains:
* image: Tensor, image in (C, H, W) format.
* instances: Instances
Other information that's included in the original dicts, such as:
* "height", "width" (int): the output resolution of the model, used in inference.
See :meth:`postprocess` for details.
Returns:
in training, dict[str: Tensor]:
mapping from a named loss to a tensor storing the loss. Used during training only.
in inference, the standard output format, described in :doc:`/tutorials/models`.
"""
images = self.preprocess_image(batched_inputs)
features = self.backbone(images.tensor)
features = {key: features[key] for key in features.keys() if key in self.head_in_features}
pred_logits = self.head(features)
# Transpose the Hi*Wi*A dimension to the middle:
#pred_logits = [permute_to_N_HWA_K(x, self.num_classes) for x in pred_logits]
if self.training:
assert not torch.jit.is_scripting(), "Not supported"
assert "annotations" in batched_inputs[0], "annotations are missing in training!"
gt_annotations = [x["annotations"] for x in batched_inputs]
losses = self.losses(pred_logits, gt_annotations)
if self.vis_period > 0:
storage = get_event_storage()
if storage.iter % self.vis_period == 0:
results = self.inference(
pred_logits, images.image_sizes
)
#self.visualize_training(batched_inputs, results)
return losses
else:
results = self.inference(pred_logits, images.image_sizes)
return results