def test_sce_equals_ce(self): # All correct predictions Y = torch.tensor([1, 2, 3], dtype=torch.long) Y_s = hard_to_soft(Y, k=4) sce = SoftCrossEntropyLoss(reduction="none") ce = nn.CrossEntropyLoss(reduction="none") for _ in range(10): Y_ps = torch.rand_like(Y_s) Y_ps = Y_ps / Y_ps.sum(dim=1).reshape(-1, 1) self.assertTrue((sce(Y_ps, Y_s) == ce(Y_ps, Y - 1)).all()) sce = SoftCrossEntropyLoss(reduction="sum") ce = nn.CrossEntropyLoss(reduction="sum") for _ in range(10): Y_ps = torch.rand_like(Y_s) Y_ps = Y_ps / Y_ps.sum(dim=1).reshape(-1, 1) self.assertAlmostEqual(sce(Y_ps, Y_s).numpy(), ce(Y_ps, Y - 1).numpy(), places=5) sce = SoftCrossEntropyLoss(reduction="elementwise_mean") ce = nn.CrossEntropyLoss(reduction="elementwise_mean") for _ in range(10): Y_ps = torch.rand_like(Y_s) Y_ps = Y_ps / Y_ps.sum(dim=1).reshape(-1, 1) self.assertAlmostEqual(sce(Y_ps, Y_s).numpy(), ce(Y_ps, Y - 1).numpy(), places=5)
def roc_auc_score(gold, probs, ignore_in_gold=[], ignore_in_pred=[]): """Compute the ROC AUC score, given the gold labels and predicted probs. Args: gold: A 1d array-like of gold labels probs: A 2d array-like of predicted probabilities ignore_in_gold: A list of labels for which elements having that gold label will be ignored. Returns: roc_auc_score: The (float) roc_auc score """ gold = arraylike_to_numpy(gold) # Filter out the ignore_in_gold (but not ignore_in_pred) # Note the current sub-functions (below) do not handle this... if len(ignore_in_pred) > 0: raise ValueError("ignore_in_pred not defined for ROC-AUC score.") keep = [x not in ignore_in_gold for x in gold] gold = gold[keep] probs = probs[keep, :] # Convert gold to one-hot indicator format, using the k inferred from probs gold_s = hard_to_soft(torch.from_numpy(gold), k=probs.shape[1]).numpy() return skm.roc_auc_score(gold_s, probs)
def _preprocess_Y(self, Y, k): """Convert Y to soft labels if necessary""" Y = Y.clone() # If hard labels, convert to soft labels if Y.dim() == 1 or Y.shape[1] == 1: Y = hard_to_soft(Y.long(), k=k) return Y
def test_hard_to_soft(self): x = torch.tensor([1,2,2,1]) target = torch.tensor([ [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], ], dtype=torch.float) self.assertTrue((hard_to_soft(x, 2) == target).sum() == torch.prod(torch.tensor(target.shape)))
def test_perfect_predictions(self): Y = torch.tensor([1, 2, 3], dtype=torch.long) Y_s = hard_to_soft(Y, k=4) sce = SoftCrossEntropyLoss() # Guess nearly perfectly Y_ps = Y_s.clone() Y_ps[Y_ps == 1] = 100 Y_ps[Y_ps == 0] = -100 self.assertAlmostEqual(sce(Y_ps, Y_s).numpy(), 0)
def test_perfect_predictions(self): Y_h = torch.tensor([1, 2, 3], dtype=torch.long) target = Y_h Y = hard_to_soft(Y_h, k=4) sce = SoftCrossEntropyLoss() # Guess nearly perfectly Y_p = Y.clone() Y_p[Y_p == 1] = 100 Y_p[Y_p == 0] = -100 self.assertAlmostEqual(sce(Y_p, Y).numpy(), 0)
def _preprocess_Y(self, Y): """Convert Y to soft labels if necessary""" # If hard labels, convert to soft labels if Y.dim() == 1 or Y.shape[1] == 1: if not isinstance(Y, torch.LongTensor): self._check(Y, typ=torch.LongTensor) # FIXME: This could fail if last class was never predicted Y = hard_to_soft(Y, k=Y.max().long()) # FIXME: This currently assumes that no model can output a # prediction of 0 (i.e., if cardinality=5, Y[0] corresponds to # class 1 instead of 0, since the latter would give the model # a 5-dim output space but a 6-dim label space) Y = Y[:,1:] return Y
def test_loss_weights(self): # All incorrect predictions Y = torch.tensor([1, 1, 2], dtype=torch.long) Y_s = hard_to_soft(Y, k=3) Y_ps = torch.tensor([[-100., 100., -100.], [-100., 100., -100.], [-100., 100., -100.]]) weight1 = torch.tensor([1, 2, 1], dtype=torch.float) weight2 = torch.tensor([10, 20, 10], dtype=torch.float) ce1 = nn.CrossEntropyLoss(weight=weight1, reduction="none") sce1 = SoftCrossEntropyLoss(weight=weight1) sce2 = SoftCrossEntropyLoss(weight=weight2) self.assertAlmostEqual(float(ce1(Y_ps, Y - 1).mean()), float(sce1(Y_ps, Y_s)), places=3) self.assertAlmostEqual(float(sce1(Y_ps, Y_s)) * 10, float(sce2(Y_ps, Y_s)), places=3)
def test_loss_weights(self): # All incorrect predictions Y_h = torch.tensor([1,1,2], dtype=torch.long) target = Y_h K_t = 3 Y = hard_to_soft(Y_h, k=K_t) Y_p = torch.tensor([ [0., -100., 100., -100.], [0., -100., 100., -100.], [0., -100., 100., -100.], ]) weight1 = torch.tensor([0,1,2,1], dtype=torch.float) weight2 = torch.tensor([0,10,20,10], dtype=torch.float) ce1 = nn.CrossEntropyLoss(weight=weight1, reduction='none') sce1 = SoftCrossEntropyLoss(weight=weight1) sce2 = SoftCrossEntropyLoss(weight=weight2) self.assertAlmostEqual( float(ce1(Y_p, target).mean()), float(sce1(Y_p, Y)), places=3) self.assertAlmostEqual( float(sce1(Y_p, Y)) * 10, float(sce2(Y_p, Y)), places=3)
def test_sce_equals_ce(self): # All correct predictions Y_h = torch.tensor([1, 2, 3], dtype=torch.long) target = Y_h Y = hard_to_soft(Y_h, k=4) sce = SoftCrossEntropyLoss(reduction='none') ce = nn.CrossEntropyLoss(reduction='none') for _ in range(10): Y_p = torch.randn(Y.shape) self.assertTrue((sce(Y_p, Y) == ce(Y_p, target)).all()) sce = SoftCrossEntropyLoss(reduction='sum') ce = nn.CrossEntropyLoss(reduction='sum') for _ in range(10): self.assertAlmostEqual(sce(Y_p, Y).numpy(), ce(Y_p, target).numpy(), places=5) sce = SoftCrossEntropyLoss(reduction='elementwise_mean') # default ce = nn.CrossEntropyLoss(reduction='elementwise_mean') for _ in range(10): self.assertAlmostEqual(sce(Y_p, Y).numpy(), ce(Y_p, target).numpy(), places=5)
def test_hard_to_soft(self): x = torch.tensor([1, 2, 2, 1]) target = torch.tensor([[1, 0], [0, 1], [0, 1], [1, 0]]) self.assertTrue((hard_to_soft(x, 2).float() == target.float() ).sum() == torch.prod(torch.tensor(target.shape)))