def roc_auc_score(preds, targets):
if preds.ndim == 1:
preds = preds[:, np.newaxis]
targets = targets[:, np.newaxis]
return sum(
sk_roc_auc_score(targets[:, i], preds[:, i])
for i in range(preds.shape[1])) / preds.shape[1]
def _sk_auroc_multilabel_multidim_prob(preds, target, num_classes, average='macro', max_fpr=None, multi_class='ovr'):
sk_preds = preds.transpose(0, 1).reshape(num_classes, -1).transpose(0, 1).numpy()
sk_target = target.transpose(0, 1).reshape(num_classes, -1).transpose(0, 1).numpy()
return sk_roc_auc_score(
y_true=sk_target,
y_score=sk_preds,
average=average,
max_fpr=max_fpr,
multi_class=multi_class,
)
def _sk_auroc_multiclass_prob(preds, target, num_classes, average='macro', max_fpr=None, multi_class='ovr'):
sk_preds = preds.reshape(-1, num_classes).numpy()
sk_target = target.view(-1).numpy()
return sk_roc_auc_score(
y_true=sk_target,
y_score=sk_preds,
average=average,
max_fpr=max_fpr,
multi_class=multi_class,
)
def _sk_auroc_binary_prob(preds,
target,
num_classes,
average='macro',
max_fpr=None,
multi_class='ovr'):
# todo: `multi_class` is unused
sk_preds = preds.view(-1).numpy()
sk_target = target.view(-1).numpy()
return sk_roc_auc_score(y_true=sk_target,
y_score=sk_preds,
average=average,
max_fpr=max_fpr)
def _sk_auroc_multilabel_prob(preds,
target,
num_classes,
average="macro",
max_fpr=None,
multi_class="ovr"):
sk_preds = preds.reshape(-1, num_classes).numpy()
sk_target = target.reshape(-1, num_classes).numpy()
return sk_roc_auc_score(
y_true=sk_target,
y_score=sk_preds,
average=average,
max_fpr=max_fpr,
multi_class=multi_class,
)
def _sk_auroc_multidim_multiclass_prob(preds,
target,
num_classes,
average="macro",
max_fpr=None,
multi_class="ovr"):
sk_preds = preds.transpose(0, 1).reshape(num_classes,
-1).transpose(0, 1).numpy()
sk_target = target.view(-1).numpy()
return sk_roc_auc_score(
y_true=sk_target,
y_score=sk_preds,
average=average,
max_fpr=max_fpr,
multi_class=multi_class,
)
def test_multiclass_auroc_against_sklearn(n_cls):
device = 'cuda' if torch.cuda.is_available() else 'cpu'
n_samples = 300
pred = torch.rand(n_samples, n_cls, device=device).softmax(dim=1)
target = torch.randint(n_cls, (n_samples, ), device=device)
# Make sure target includes all class labels so that multiclass AUROC is defined
target[10:10 + n_cls] = torch.arange(n_cls)
pl_score = multiclass_auroc(pred, target)
# For the binary case, sklearn expects an (n_samples,) array of probabilities of
# the positive class
pred = pred[:, 1] if n_cls == 2 else pred
sk_score = sk_roc_auc_score(target.cpu().detach().numpy(),
pred.cpu().detach().numpy(),
multi_class="ovr")
sk_score = torch.tensor(sk_score, dtype=torch.float, device=device)
assert torch.allclose(sk_score, pl_score)
def roc_auc_score(gold, pred):
return sk_roc_auc_score(gold, pred), 0.0
