def test_sce_equals_ce(self):
# All correct predictions
Y = torch.tensor([1, 2, 3], dtype=torch.long)
Y_s = hard_to_soft(Y, k=4)
sce = SoftCrossEntropyLoss(reduction="none")
ce = nn.CrossEntropyLoss(reduction="none")
for _ in range(10):
Y_ps = torch.rand_like(Y_s)
Y_ps = Y_ps / Y_ps.sum(dim=1).reshape(-1, 1)
self.assertTrue((sce(Y_ps, Y_s) == ce(Y_ps, Y - 1)).all())
sce = SoftCrossEntropyLoss(reduction="sum")
ce = nn.CrossEntropyLoss(reduction="sum")
for _ in range(10):
Y_ps = torch.rand_like(Y_s)
Y_ps = Y_ps / Y_ps.sum(dim=1).reshape(-1, 1)
self.assertAlmostEqual(sce(Y_ps, Y_s).numpy(),
ce(Y_ps, Y - 1).numpy(),
places=5)
sce = SoftCrossEntropyLoss(reduction="elementwise_mean")
ce = nn.CrossEntropyLoss(reduction="elementwise_mean")
for _ in range(10):
Y_ps = torch.rand_like(Y_s)
Y_ps = Y_ps / Y_ps.sum(dim=1).reshape(-1, 1)
self.assertAlmostEqual(sce(Y_ps, Y_s).numpy(),
ce(Y_ps, Y - 1).numpy(),
places=5)
def roc_auc_score(gold, probs, ignore_in_gold=[], ignore_in_pred=[]):
"""Compute the ROC AUC score, given the gold labels and predicted probs.
Args:
gold: A 1d array-like of gold labels
probs: A 2d array-like of predicted probabilities
ignore_in_gold: A list of labels for which elements having that gold
label will be ignored.
Returns:
roc_auc_score: The (float) roc_auc score
"""
gold = arraylike_to_numpy(gold)
# Filter out the ignore_in_gold (but not ignore_in_pred)
# Note the current sub-functions (below) do not handle this...
if len(ignore_in_pred) > 0:
raise ValueError("ignore_in_pred not defined for ROC-AUC score.")
keep = [x not in ignore_in_gold for x in gold]
gold = gold[keep]
probs = probs[keep, :]
# Convert gold to one-hot indicator format, using the k inferred from probs
gold_s = hard_to_soft(torch.from_numpy(gold), k=probs.shape[1]).numpy()
return skm.roc_auc_score(gold_s, probs)
def _preprocess_Y(self, Y, k):
"""Convert Y to soft labels if necessary"""
Y = Y.clone()
# If hard labels, convert to soft labels
if Y.dim() == 1 or Y.shape[1] == 1:
Y = hard_to_soft(Y.long(), k=k)
return Y
def test_hard_to_soft(self):
x = torch.tensor([1,2,2,1])
target = torch.tensor([
[0, 1, 0],
[0, 0, 1],
[0, 0, 1],
[0, 1, 0],
], dtype=torch.float)
self.assertTrue((hard_to_soft(x, 2) == target).sum()
== torch.prod(torch.tensor(target.shape)))
def test_perfect_predictions(self):
Y = torch.tensor([1, 2, 3], dtype=torch.long)
Y_s = hard_to_soft(Y, k=4)
sce = SoftCrossEntropyLoss()
# Guess nearly perfectly
Y_ps = Y_s.clone()
Y_ps[Y_ps == 1] = 100
Y_ps[Y_ps == 0] = -100
self.assertAlmostEqual(sce(Y_ps, Y_s).numpy(), 0)
def test_perfect_predictions(self):
Y_h = torch.tensor([1, 2, 3], dtype=torch.long)
target = Y_h
Y = hard_to_soft(Y_h, k=4)
sce = SoftCrossEntropyLoss()
# Guess nearly perfectly
Y_p = Y.clone()
Y_p[Y_p == 1] = 100
Y_p[Y_p == 0] = -100
self.assertAlmostEqual(sce(Y_p, Y).numpy(), 0)
def _preprocess_Y(self, Y):
"""Convert Y to soft labels if necessary"""
# If hard labels, convert to soft labels
if Y.dim() == 1 or Y.shape[1] == 1:
if not isinstance(Y, torch.LongTensor):
self._check(Y, typ=torch.LongTensor)
# FIXME: This could fail if last class was never predicted
Y = hard_to_soft(Y, k=Y.max().long())
# FIXME: This currently assumes that no model can output a
# prediction of 0 (i.e., if cardinality=5, Y[0] corresponds to
# class 1 instead of 0, since the latter would give the model
# a 5-dim output space but a 6-dim label space)
Y = Y[:,1:]
return Y
def test_loss_weights(self):
# All incorrect predictions
Y = torch.tensor([1, 1, 2], dtype=torch.long)
Y_s = hard_to_soft(Y, k=3)
Y_ps = torch.tensor([[-100., 100., -100.], [-100., 100., -100.],
[-100., 100., -100.]])
weight1 = torch.tensor([1, 2, 1], dtype=torch.float)
weight2 = torch.tensor([10, 20, 10], dtype=torch.float)
ce1 = nn.CrossEntropyLoss(weight=weight1, reduction="none")
sce1 = SoftCrossEntropyLoss(weight=weight1)
sce2 = SoftCrossEntropyLoss(weight=weight2)
self.assertAlmostEqual(float(ce1(Y_ps, Y - 1).mean()),
float(sce1(Y_ps, Y_s)),
places=3)
self.assertAlmostEqual(float(sce1(Y_ps, Y_s)) * 10,
float(sce2(Y_ps, Y_s)),
places=3)
def test_loss_weights(self):
# All incorrect predictions
Y_h = torch.tensor([1,1,2], dtype=torch.long)
target = Y_h
K_t = 3
Y = hard_to_soft(Y_h, k=K_t)
Y_p = torch.tensor([
[0., -100., 100., -100.],
[0., -100., 100., -100.],
[0., -100., 100., -100.],
])
weight1 = torch.tensor([0,1,2,1], dtype=torch.float)
weight2 = torch.tensor([0,10,20,10], dtype=torch.float)
ce1 = nn.CrossEntropyLoss(weight=weight1, reduction='none')
sce1 = SoftCrossEntropyLoss(weight=weight1)
sce2 = SoftCrossEntropyLoss(weight=weight2)
self.assertAlmostEqual(
float(ce1(Y_p, target).mean()), float(sce1(Y_p, Y)), places=3)
self.assertAlmostEqual(
float(sce1(Y_p, Y)) * 10, float(sce2(Y_p, Y)), places=3)
def test_sce_equals_ce(self):
# All correct predictions
Y_h = torch.tensor([1, 2, 3], dtype=torch.long)
target = Y_h
Y = hard_to_soft(Y_h, k=4)
sce = SoftCrossEntropyLoss(reduction='none')
ce = nn.CrossEntropyLoss(reduction='none')
for _ in range(10):
Y_p = torch.randn(Y.shape)
self.assertTrue((sce(Y_p, Y) == ce(Y_p, target)).all())
sce = SoftCrossEntropyLoss(reduction='sum')
ce = nn.CrossEntropyLoss(reduction='sum')
for _ in range(10):
self.assertAlmostEqual(sce(Y_p, Y).numpy(), ce(Y_p, target).numpy(),
places=5)
sce = SoftCrossEntropyLoss(reduction='elementwise_mean') # default
ce = nn.CrossEntropyLoss(reduction='elementwise_mean')
for _ in range(10):
self.assertAlmostEqual(sce(Y_p, Y).numpy(), ce(Y_p, target).numpy(),
places=5)
def test_hard_to_soft(self):
x = torch.tensor([1, 2, 2, 1])
target = torch.tensor([[1, 0], [0, 1], [0, 1], [1, 0]])
self.assertTrue((hard_to_soft(x, 2).float() == target.float()
).sum() == torch.prod(torch.tensor(target.shape)))