def visualize_training(self, batched_inputs, results): #from herbarium.utils.visualizer import Visualizer assert len(batched_inputs) == len(results), "Cannot visualize inputs and results of different sizes" storage = get_event_storage() y_true = batched_inputs["annotations"] #v_gt = Visualizer(gt_annotation, results) vis_name = f"per-batch f1 score" storage.put_scalar(vis_name, f1_score(y_true, results))
def _write_metrics( self, loss_dict: Dict[str, torch.Tensor], data_time: float, controller_time: float, base_time: float, prefix: str = "", ): """ Args: loss_dict (dict): dict of scalar losses data_time (float): time taken by the dataloader iteration """ metrics_dict = { k: v.detach().cpu().item() for k, v in loss_dict.items() } metrics_dict["data_time"] = data_time metrics_dict["controller_time"] = controller_time metrics_dict["base_time"] = base_time # Gather metrics among all workers for logging # This assumes we do DDP-style training, which is currently the only # supported method in herbarium. all_metrics_dict = comm.gather(metrics_dict) if comm.is_main_process(): storage = get_event_storage() # data_time among workers can have high variance. The actual latency # caused by data_time is the maximum among workers. data_time = np.max([x.pop("data_time") for x in all_metrics_dict]) controller_time = np.max( [x.pop("controller_time") for x in all_metrics_dict]) base_time = np.max([x.pop("base_time") for x in all_metrics_dict]) storage.put_scalar("data_time", data_time) storage.put_scalar("controller_time", controller_time) storage.put_scalar("base_time", base_time) # average the rest metrics metrics_dict = { k: np.mean([x[k] for x in all_metrics_dict]) for k in all_metrics_dict[0].keys() } total_losses_reduced = sum(metrics_dict.values()) if not np.isfinite(total_losses_reduced): raise FloatingPointError( f"Loss became infinite or NaN at iteration={self.iter}!\n" f"loss_dict = {metrics_dict}") storage.put_scalar("{}total_loss".format(prefix), total_losses_reduced) if len(metrics_dict) > 0: storage.put_scalars(**metrics_dict)
def forward(self, batched_inputs: Tuple[Dict[str, Tensor]]): """ Args: batched_inputs: a list, batched outputs of :class:`DatasetMapper` . Each item in the list contains the inputs for one image. For now, each item in the list is a dict that contains: * image: Tensor, image in (C, H, W) format. * instances: Instances Other information that's included in the original dicts, such as: * "height", "width" (int): the output resolution of the model, used in inference. See :meth:`postprocess` for details. Returns: in training, dict[str: Tensor]: mapping from a named loss to a tensor storing the loss. Used during training only. in inference, the standard output format, described in :doc:`/tutorials/models`. """ images = self.preprocess_image(batched_inputs) features = self.backbone(images.tensor) features = {key: features[key] for key in features.keys() if key in self.head_in_features} pred_logits = self.head(features) # Transpose the Hi*Wi*A dimension to the middle: #pred_logits = [permute_to_N_HWA_K(x, self.num_classes) for x in pred_logits] if self.training: assert not torch.jit.is_scripting(), "Not supported" assert "annotations" in batched_inputs[0], "annotations are missing in training!" gt_annotations = [x["annotations"] for x in batched_inputs] losses = self.losses(pred_logits, gt_annotations) if self.vis_period > 0: storage = get_event_storage() if storage.iter % self.vis_period == 0: results = self.inference( pred_logits, images.image_sizes ) #self.visualize_training(batched_inputs, results) return losses else: results = self.inference(pred_logits, images.image_sizes) return results