def compute(self) -> Tuple[torch.Tensor, float, float, float]: """Computes the AUC metric based on saved statistics.""" targets = torch.cat(self.targets) scores = torch.cat(self.scores) # ddp hotfix, could be done better # but metric must handle DDP on it's own if self._ddp_backend == "xla": # if you have "RuntimeError: Aborted: Session XXX is not found" here # please, ask Google for a more powerful TPU setup ;) device = get_device() scores = xm.all_gather(scores.to(device)).cpu().detach() targets = xm.all_gather(targets.to(device)).cpu().detach() elif self._ddp_backend == "ddp": scores = torch.cat(all_gather(scores)) targets = torch.cat(all_gather(targets)) scores, targets, _, _ = process_multilabel_components( outputs=scores, targets=targets ) per_class = auc(scores=scores, targets=targets) micro = binary_auc(scores=scores.view(-1), targets=targets.view(-1))[0] macro = per_class.mean().item() weights = targets.sum(axis=0) / len(targets) weighted = (per_class * weights).sum().item() if self.compute_per_class_metrics: return per_class, micro, macro, weighted else: return [], micro, macro, weighted
def compute(self) -> torch.Tensor: """Computes the AUC metric based on saved statistics.""" targets = torch.cat(self.targets) scores = torch.cat(self.scores) # @TODO: ddp hotfix, could be done better if self._is_ddp: scores = torch.cat(all_gather(scores)) targets = torch.cat(all_gather(targets)) score = auc(outputs=scores, targets=targets) return score
def compute(self) -> Any: """ Returns: Confusion matrix of K rows and K columns, where rows corresponds to ground-truth targets and columns corresponds to predicted targets. """ # ddp hotfix, could be done better # but metric must handle DDP on it's own if self._ddp_backend == "xla": # if you have "RuntimeError: Aborted: Session XXX is not found" here # please, ask Google for a more powerful TPU setup ;) device = get_device() value = torch.tensor([self.conf], device=device) self.conf = xm.all_gather(value).sum(0).cpu().detach().numpy() elif self._ddp_backend == "ddp": value: List[np.ndarray] = all_gather(self.conf) value: np.ndarray = np.sum(np.stack(value, axis=0), axis=0) self.conf = value if self.normalized: conf = self.conf.astype(np.float32) return conf / conf.sum(1).clip(min=1e-12)[:, None] else: return self.conf
def compute(self) -> Tuple[float, float, float]: """ Compute metrics with accumulated statistics Returns: tuple of metrics: precision, recall, f1 score """ # ddp hotfix, could be done better # but metric must handle DDP on it's own if self._ddp_backend == "xla": self.statistics = { k: xm.mesh_reduce(k, v, np.sum) for k, v in self.statistics.items() } elif self._ddp_backend == "ddp": for key in self.statistics: value: List[int] = all_gather(self.statistics[key]) value: int = sum(value) self.statistics[key] = value precision_value, recall_value, f1_value = get_binary_metrics( tp=self.statistics["tp"], fp=self.statistics["fp"], fn=self.statistics["fn"], zero_division=self.zero_division, ) return precision_value, recall_value, f1_value
def compute(self) -> Any: """ Compute precision, recall, f1 score and support. Compute micro, macro and weighted average for the metrics. Returns: list of aggregated metrics: per-class, micro, macro and weighted averaging of precision, recall, f1 score and support metrics """ # ddp hotfix, could be done better # but metric must handle DDP on it's own if self._ddp_backend == "xla": device = get_device() for key in self.statistics: key_statistics = torch.tensor([self.statistics[key]], device=device) key_statistics = xm.all_gather(key_statistics).sum( dim=0).cpu().numpy() self.statistics[key] = key_statistics elif self._ddp_backend == "ddp": for key in self.statistics: value: List[np.ndarray] = all_gather(self.statistics[key]) value: np.ndarray = np.sum(np.vstack(value), axis=0) self.statistics[key] = value per_class, micro, macro, weighted = get_aggregated_metrics( tp=self.statistics["tp"], fp=self.statistics["fp"], fn=self.statistics["fn"], support=self.statistics["support"], zero_division=self.zero_division, ) return per_class, micro, macro, weighted
def compute(self) -> Tuple[torch.Tensor, float, float, float]: """Computes the AUC metric based on saved statistics.""" targets = torch.cat(self.targets) scores = torch.cat(self.scores) # @TODO: ddp hotfix, could be done better if self._is_ddp: scores = torch.cat(all_gather(scores)) targets = torch.cat(all_gather(targets)) scores, targets, _ = process_multilabel_components(outputs=scores, targets=targets) per_class = auc(scores=scores, targets=targets) micro = binary_auc(scores=scores.view(-1), targets=targets.view(-1))[0] macro = per_class.mean().item() weights = targets.sum(axis=0) / len(targets) weighted = (per_class * weights).sum().item() return per_class, micro, macro, weighted
def _all_gather(rank, world_size): _setup(rank, world_size) to_gather = torch.ones(3, dtype=torch.int) * (rank + 1) # use cpu tensors actual = all_gather(to_gather) actual = torch.cat(actual) expected = torch.cat([torch.ones(3, dtype=torch.int) * (i + 1) for i in range(world_size)]) assert torch.all(actual.eq(expected)) _cleanup()
def compute(self) -> Any: """ Returns: Confusion matrix of K rows and K columns, where rows corresponds to ground-truth targets and columns corresponds to predicted targets. """ if self._is_ddp: value: List[np.ndarray] = all_gather(self.conf) value: np.ndarray = np.sum(np.stack(value, axis=0), axis=0) self.conf = value if self.normalized: conf = self.conf.astype(np.float32) return conf / conf.sum(1).clip(min=1e-12)[:, None] else: return self.conf
def compute(self): """ Compute metrics with accumulated statistics Returns: tuple of metrics: per_class, micro_metric, macro_metric, weighted_metric(None if weights is None) """ per_class = [] total_statistics = {} macro_metric = 0 weighted_metric = 0 # ddp hotfix, could be done better # but metric must handle DDP on it's own # TODO: optimise speed if self._ddp_backend == "xla": device = get_device() for _, statistics in self.statistics.items(): for key in statistics: value = torch.tensor([statistics[key]], device=device) statistics[key] = xm.all_gather(value).sum(dim=0) elif self._ddp_backend == "ddp": for _, statistics in self.statistics.items(): for key in statistics: value: List[torch.Tensor] = all_gather(statistics[key]) value: torch.Tensor = torch.sum(torch.vstack(value), dim=0) statistics[key] = value for class_idx, statistics in self.statistics.items(): value = self.metric_fn(**statistics) per_class.append(value) macro_metric += value if self.weights is not None: weighted_metric += value * self.weights[class_idx] for stats_name, value in statistics.items(): total_statistics[stats_name] = ( total_statistics.get(stats_name, 0) + value) macro_metric /= len(self.statistics) micro_metric = self.metric_fn(**total_statistics) if self.weights is None: weighted_metric = None if self.compute_per_class_metrics: return per_class, micro_metric, macro_metric, weighted_metric else: return [], micro_metric, macro_metric, weighted_metric
def compute_key_value(self) -> Dict[str, torch.Tensor]: """ Compute segmentation metric for all data and return results in key-value format Returns: dict of metrics, including micro, macro and weighted (if weights were given) metrics """ metrics = {} total_statistics = {} macro_metric = 0 weighted_metric = 0 # @TODO: ddp hotfix, could be done better if self._is_ddp: for _, statistics in self.statistics.items(): for key in statistics: device = statistics[key].device value: List[torch.Tensor] = all_gather( statistics[key].cpu()) value: torch.Tensor = torch.sum(torch.vstack(value), dim=0).to(device) statistics[key] = value for class_idx, statistics in self.statistics.items(): value = self.metric_fn(**statistics) macro_metric += value if self.weights is not None: weighted_metric += value * self.weights[class_idx] metrics[ f"{self.prefix}{self.metric_name}{self.suffix}/{self.class_names[class_idx]}"] = value for stats_name, value in statistics.items(): total_statistics[stats_name] = total_statistics.get( stats_name, 0) + value macro_metric /= len(self.statistics) micro_metric = self.metric_fn(**total_statistics) metrics[ f"{self.prefix}{self.metric_name}{self.suffix}/_micro"] = micro_metric metrics[f"{self.prefix}{self.metric_name}{self.suffix}"] = macro_metric metrics[ f"{self.prefix}{self.metric_name}{self.suffix}/_macro"] = macro_metric if self.weights is not None: metrics[ f"{self.prefix}{self.metric_name}{self.suffix}/_weighted"] = weighted_metric # convert torch.Tensor to float # metrics = {k: float(v) for k, v in metrics.items()} return metrics
def compute(self) -> Tuple[float, float, float]: """ Compute metrics with accumulated statistics Returns: tuple of metrics: precision, recall, f1 score """ # @TODO: ddp hotfix, could be done better if self._is_ddp: for key in self.statistics: value: List[float] = all_gather(self.statistics[key]) value: float = sum(value) self.statistics[key] = value precision_value, recall_value, f1_value = get_binary_metrics( tp=self.statistics["tp"], fp=self.statistics["fp"], fn=self.statistics["fn"], zero_division=self.zero_division, ) return precision_value, recall_value, f1_value
def compute_key_value(self) -> Dict[str, float]: """ Compute precision, recall, f1 score and support. Compute micro, macro and weighted average for the metrics. Returns: dict of metrics """ # @TODO: ddp hotfix, could be done better if self._is_ddp: for key in self.statistics: value: List[np.ndarray] = all_gather(self.statistics[key]) value: np.ndarray = np.sum(np.vstack(value), axis=0) self.statistics[key] = value per_class, micro, macro, weighted = self.compute() metrics = self._convert_metrics_to_kv(per_class=per_class, micro=micro, macro=macro, weighted=weighted) return metrics