def validation_epoch_end(self, outputs):
# each process stores features separately, so gather them together to calculate FID over the full distribution
real_features = dim_zero_cat(self.fid.real_features)
real_features_list = [
torch.empty_like(real_features)
for _ in range(dist.get_world_size())
]
dist.all_gather(real_features_list, real_features)
real_features = dim_zero_cat(real_features_list)
fake_features = dim_zero_cat(self.fid.fake_features)
fake_features_list = [
torch.empty_like(fake_features)
for _ in range(dist.get_world_size())
]
dist.all_gather(fake_features_list, fake_features)
fake_features = dim_zero_cat(fake_features_list)
rank = dist.get_rank()
if rank == 0:
fid = calculate_fid(real_features,
fake_features) # returned as numpy array
fid = torch.tensor(
[float(fid)],
device=self.device) # but we need a torch tensor for DDP
print('')
print('FID with {} samples: {}'.format(len(real_features), fid))
print('')
else:
fid = torch.tensor([0.0], device=self.device)
# share the result with all GPUs so that the checkpointing function doesn't crash
dist.broadcast(tensor=fid, src=0)
self.log('metrics/fid', fid, rank_zero_only=True)
def compute(
self
) -> Union[Tuple[Tensor, Tensor, Tensor], Tuple[List[Tensor], List[Tensor],
List[Tensor]]]:
"""
Compute the precision-recall curve
Returns:
3-element tuple containing
precision:
tensor where element i is the precision of predictions with
score >= thresholds[i] and the last element is 1.
If multiclass, this is a list of such tensors, one for each class.
recall:
tensor where element i is the recall of predictions with
score >= thresholds[i] and the last element is 0.
If multiclass, this is a list of such tensors, one for each class.
thresholds:
Thresholds used for computing precision/recall scores
"""
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
return _precision_recall_curve_compute(preds, target, self.num_classes,
self.pos_label)
def compute(self):
"""
Computes pearson correlation coefficient over state.
"""
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
return _pearson_corrcoef_compute(preds, target)
def compute(self):
"""
Computes spearmans correlation coefficient
"""
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
return _spearman_corrcoef_compute(preds, target)
def compute(self) -> Tensor:
"""Computes explained variance over state."""
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
return _ssim_compute(
preds, target, self.kernel_size, self.sigma, self.reduction, self.data_range, self.k1, self.k2
)
def compute(
self
) -> Union[Tuple[Tensor, Tensor, Tensor], Tuple[List[Tensor], List[Tensor],
List[Tensor]]]:
"""Compute the precision-recall curve.
Returns:
3-element tuple containing
precision:
tensor where element ``i`` is the precision of predictions with
``score >= thresholds[i]`` and the last element is 1.
If multiclass, this is a list of such tensors, one for each class.
recall:
tensor where element ``i`` is the recall of predictions with
``score >= thresholds[i]`` and the last element is 0.
If multiclass, this is a list of such tensors, one for each class.
thresholds:
Thresholds used for computing precision/recall scores
"""
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
if not self.num_classes:
raise ValueError(
f"`num_classes` bas to be positive number, but got {self.num_classes}"
)
return _precision_recall_curve_compute(preds, target, self.num_classes,
self.pos_label)
def compute(self) -> Tensor:
"""
Computes AUC based on inputs passed in to ``update`` previously.
"""
x = dim_zero_cat(self.x)
y = dim_zero_cat(self.y)
return _auc_compute(x, y, reorder=self.reorder)
def compute(self) -> Tuple[Tensor, Tensor]:
"""Calculate KID score based on accumulated extracted features from the two distributions. Returns a tuple
of mean and standard deviation of KID scores calculated on subsets of extracted features.
Implementation inspired by `Fid Score`_
"""
real_features = dim_zero_cat(self.real_features)
fake_features = dim_zero_cat(self.fake_features)
n_samples_real = real_features.shape[0]
if n_samples_real < self.subset_size:
raise ValueError("Argument `subset_size` should be smaller than the number of samples")
n_samples_fake = fake_features.shape[0]
if n_samples_fake < self.subset_size:
raise ValueError("Argument `subset_size` should be smaller than the number of samples")
kid_scores_ = []
for _ in range(self.subsets):
perm = torch.randperm(n_samples_real)
f_real = real_features[perm[: self.subset_size]]
perm = torch.randperm(n_samples_fake)
f_fake = fake_features[perm[: self.subset_size]]
o = poly_mmd(f_real, f_fake, self.degree, self.gamma, self.coef)
kid_scores_.append(o)
kid_scores = torch.stack(kid_scores_)
return kid_scores.mean(), kid_scores.std(unbiased=False)
def compute(self) -> Tensor:
"""Computes calibration error across all confidences and accuracies.
Returns:
Tensor: Calibration error across previously collected examples.
"""
confidences = dim_zero_cat(self.confidences)
accuracies = dim_zero_cat(self.accuracies)
return _ce_compute(confidences, accuracies, self.bin_boundaries, norm=self.norm)
def compute(self) -> Union[Tensor, List[Tensor]]:
"""Compute the average precision score.
Returns:
tensor with average precision. If multiclass will return list
of such tensors, one for each class
"""
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
if not self.num_classes:
raise ValueError(f"`num_classes` bas to be positive number, but got {self.num_classes}")
return _average_precision_compute(preds, target, self.num_classes, self.pos_label, self.average)
def compute(self) -> Union[Tensor, List[Tensor]]:
"""
Compute the average precision score
Returns:
tensor with average precision. If multiclass will return list
of such tensors, one for each class
"""
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
return _average_precision_compute(preds, target, self.num_classes,
self.pos_label)
def compute(self) -> Tensor:
"""Computes AUROC based on inputs passed in to ``update`` previously."""
if not self.mode:
raise RuntimeError("You have to have determined mode.")
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
return _auroc_compute(
preds,
target,
self.mode,
self.num_classes,
self.pos_label,
self.average,
self.max_fpr,
)
def compute(self) -> Tensor:
"""
Computes AUROC based on inputs passed in to ``update`` previously.
"""
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
return _auroc_compute(
preds,
target,
self.mode,
self.num_classes,
self.pos_label,
self.average,
self.max_fpr,
)
def _sync_dist(self,
dist_sync_fn: Callable = gather_all_tensors,
process_group: Optional[Any] = None) -> None:
input_dict = {attr: getattr(self, attr) for attr in self._reductions}
for attr, reduction_fn in self._reductions.items():
# pre-concatenate metric states that are lists to reduce number of all_gather operations
if reduction_fn == dim_zero_cat and isinstance(
input_dict[attr], list) and len(input_dict[attr]) > 1:
input_dict[attr] = [dim_zero_cat(input_dict[attr])]
output_dict = apply_to_collection(
input_dict,
Tensor,
dist_sync_fn,
group=process_group or self.process_group,
)
for attr, reduction_fn in self._reductions.items():
# pre-processing ops (stack or flatten for inputs)
if isinstance(output_dict[attr][0], Tensor):
output_dict[attr] = torch.stack(output_dict[attr])
elif isinstance(output_dict[attr][0], list):
output_dict[attr] = _flatten(output_dict[attr])
if not (callable(reduction_fn) or reduction_fn is None):
raise TypeError('reduction_fn must be callable or None')
reduced = reduction_fn(
output_dict[attr]
) if reduction_fn is not None else output_dict[attr]
setattr(self, attr, reduced)
def compute(self) -> Tuple[Tensor, Tensor]:
features = dim_zero_cat(self.features)
# random permute the features
idx = torch.randperm(features.shape[0])
features = features[idx]
# calculate probs and logits
prob = features.softmax(dim=1)
log_prob = features.log_softmax(dim=1)
# split into groups
prob = prob.chunk(self.splits, dim=0)
log_prob = log_prob.chunk(self.splits, dim=0)
# calculate score per split
mean_prob = [p.mean(dim=0, keepdim=True) for p in prob]
kl_ = [
p * (log_p - m_p.log())
for p, log_p, m_p in zip(prob, log_prob, mean_prob)
]
kl_ = [k.sum(dim=1).mean().exp() for k in kl_]
kl = torch.stack(kl_)
# return mean and std
return kl.mean(), kl.std()
def _sync_dist(self, dist_sync_fn=gather_all_tensors):
input_dict = {
attr: getattr(self, attr)
for attr in self._reductions.keys()
}
for attr, reduction_fn in self._reductions.items():
# pre-concatenate metric states that are lists to reduce number of all_gather operations
if reduction_fn == dim_zero_cat and isinstance(
input_dict[attr], list) and len(input_dict[attr]) > 1:
input_dict[attr] = [dim_zero_cat(input_dict[attr])]
output_dict = apply_to_collection(
input_dict,
Tensor,
dist_sync_fn,
group=self.process_group,
)
for attr, reduction_fn in self._reductions.items():
# pre-processing ops (stack or flatten for inputs)
if isinstance(output_dict[attr][0], Tensor):
output_dict[attr] = torch.stack(output_dict[attr])
elif isinstance(output_dict[attr][0], list):
output_dict[attr] = _flatten(output_dict[attr])
assert isinstance(reduction_fn, Callable) or reduction_fn is None
reduced = reduction_fn(
output_dict[attr]
) if reduction_fn is not None else output_dict[attr]
setattr(self, attr, reduced)
def compute(self) -> Tensor:
"""Computes explained variance over state."""
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
return _ssim_compute(
preds,
target,
self.gaussian_kernel,
self.sigma,
self.kernel_size,
self.reduction,
self.data_range,
self.k1,
self.k2,
self.return_full_image,
self.return_contrast_sensitivity,
)
def compute(self) -> Tensor:
"""Computes explained variance over state."""
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
return _multiscale_ssim_compute(
preds,
target,
self.gaussian_kernel,
self.sigma,
self.kernel_size,
self.reduction,
self.data_range,
self.k1,
self.k2,
self.betas,
self.normalize,
)
def compute(self) -> Tensor:
"""Calculate FID score based on accumulated extracted features from the two distributions."""
real_features = dim_zero_cat(self.real_features)
fake_features = dim_zero_cat(self.fake_features)
# computation is extremely sensitive so it needs to happen in double precision
orig_dtype = real_features.dtype
real_features = real_features.double()
fake_features = fake_features.double()
# calculate mean and covariance
n = real_features.shape[0]
mean1 = real_features.mean(dim=0)
mean2 = fake_features.mean(dim=0)
diff1 = real_features - mean1
diff2 = fake_features - mean2
cov1 = 1.0 / (n - 1) * diff1.t().mm(diff1)
cov2 = 1.0 / (n - 1) * diff2.t().mm(diff2)
# compute fid
return _compute_fid(mean1, cov1, mean2, cov2).to(orig_dtype)
def compute(self) -> Tensor:
"""Computes and returns spectral distortion index."""
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
return _spectral_distortion_index_compute(preds, target, self.p,
self.reduction)
def compute(self) -> Tensor:
"""Computes Spearman's correlation coefficient."""
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
return _spearman_corrcoef_compute(preds, target)
def compute(self) -> Tensor:
measures = dim_zero_cat(self.measures) if self.reduction is None or self.reduction == 'none' else self.measures
return _kld_compute(measures, self.total, self.reduction)
def compute(self) -> Tensor:
"""Computes explained variance over state."""
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
return _ergas_compute(preds, target, self.ratio, self.reduction)
def compute(self) -> Tensor:
"""Compute the aggregated value."""
if isinstance(self.value, list) and self.value:
return dim_zero_cat(self.value)
return self.value
def compute(self) -> Tensor:
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
return _cosine_similarity_compute(preds, target, self.reduction)
def compute(self) -> Tensor:
"""Computes spectra over state."""
preds = dim_zero_cat(self.preds)
target = dim_zero_cat(self.target)
return _sam_compute(preds, target, self.reduction)