def test_meter_get_set_classy_state_test(self): # In this test we update meter0 with model_output0 & target0 # and we update meter1 with model_output1 & target1 then # transfer the state from meter1 to meter0 and validate they # give same expected value. # # Expected value is the expected value of meter1 For this test # to work, top-1 / top-2 values of meter0 / meter1 should be # different meters = [ PrecisionAtKMeter(topk=[1, 2]), PrecisionAtKMeter(topk=[1, 2]) ] model_outputs = [ torch.tensor([[0.05, 0.4, 0.05], [0.2, 0.65, 0.15], [0.33, 0.33, 0.34]]), torch.tensor([[0.05, 0.4, 0.05], [0.15, 0.65, 0.2], [0.4, 0.2, 0.4]]), ] targets = [ torch.tensor([[0, 1, 0], [1, 0, 0], [1, 1, 0]]), torch.tensor([[0, 1, 0], [0, 1, 0], [0, 1, 0]]), ] # Second update's expected value expected_value = {"top_1": 2 / 3.0, "top_2": 2 / 6.0} self.meter_get_set_classy_state_test(meters, model_outputs, targets, expected_value)
def test_double_meter_update_and_reset(self): meter = PrecisionAtKMeter(topk=[1, 2]) # Batchsize = 3, num classes = 3, score is probability of class model_outputs = [ torch.tensor([[0.3, 0.4, 0.3], [0.2, 0.65, 0.15], [0.33, 0.33, 0.34]]), torch.tensor([[0.05, 0.4, 0.05], [0.15, 0.65, 0.2], [0.4, 0.2, 0.4]]), ] # One-hot encoding, 1 = positive for class # batch-1: sample-1: 1, sample-2: 0, sample-3: 0,1,2 # batch-2: sample-1: 1, sample-2: 1, sample-3: 1 targets = [ torch.tensor([[0, 1, 0], [1, 0, 0], [1, 1, 1]]), torch.tensor([[0, 1, 0], [0, 1, 0], [0, 1, 0]]), ] # First batch has top-1 precision of 2/3.0, top-2 precision of 4/6.0 # Second batch has top-1 precision of 2/3.0, top-2 precision of 2/6.0 expected_value = {"top_1": 4 / 6.0, "top_2": 6 / 12.0} self.meter_update_and_reset_test(meter, model_outputs, targets, expected_value)
def test_non_onehot_target(self): """ This test verifies that the meter works as expected on a single update + reset + same single update. """ meter = PrecisionAtKMeter(topk=[1, 2], target_is_one_hot=False, num_classes=3) # Batchsize = 2, num classes = 3, score is probability of class model_outputs = [ torch.tensor([[0.05, 0.4, 0.05], [0.15, 0.65, 0.2], [0.4, 0.2, 0.4]]), torch.tensor([[0.2, 0.4, 0.4], [0.2, 0.65, 0.15], [0.1, 0.8, 0.1]]), ] # One-hot encoding, 1 = positive for class targets = [ torch.tensor([[1], [1], [1]]), # [[0, 1, 0], [0, 1, 0], [0, 1, 0]] torch.tensor([[0], [1], [2]]), # [[1, 0, 0], [0, 1, 0], [0, 0, 1]] ] # Note for ties, we select randomly, so we should not use ambiguous ties # First batch has top-1 precision of 2/3.0, top-2 precision of 2/6.0 # Second batch has top-1 precision of 1/3.0, top-2 precision of 1/6.0 expected_value = {"top_1": 3 / 6.0, "top_2": 3 / 12.0} self.meter_update_and_reset_test(meter, model_outputs, targets, expected_value)
def test_meter_invalid_model_output(self): meter = PrecisionAtKMeter(topk=[1, 2]) # This model output has 3 dimensions instead of expected 2 model_output = torch.tensor([[[0.33, 0.33, 0.34], [1, 2, 3]], [[-1, -3, -4], [-10, -90, -100]]]) target = torch.tensor([[0, 1, 0], [1, 0, 0], [1, 1, 1]]) self.meter_invalid_meter_input_test(meter, model_output, target)
def test_meter_invalid_topk(self): meter = PrecisionAtKMeter(topk=[1, 5]) model_output = torch.tensor([ [0.2, 0.4, 0.4], # top-1: 1/2, top-2: 1/2 [0.2, 0.65, 0.15], # top-1: 1, top-2: 1/0 [0.33, 0.33, 0.34], # top-1: 2, top-2: 2/0/1 ]) target = torch.tensor([[0, 1, 0], [1, 0, 0], [1, 1, 1]]) self.meter_invalid_meter_input_test(meter, model_output, target)
def test_meter_distributed(self): # Meter0 will execute on one process, Meter1 on the other meters = [ PrecisionAtKMeter(topk=[1, 2]), PrecisionAtKMeter(topk=[1, 2]) ] # Batchsize = 3, num classes = 3, score is probability of class model_outputs = [ torch.tensor([[0.3, 0.4, 0.3], [0.2, 0.65, 0.15], [0.33, 0.33, 0.34]]), # Meter 0 torch.tensor([[0.05, 0.4, 0.05], [0.15, 0.65, 0.2], [0.4, 0.2, 0.4]]), # Meter 1 torch.tensor([[0.3, 0.4, 0.3], [0.2, 0.65, 0.15], [0.33, 0.33, 0.34]]), # Meter 0 torch.tensor([[0.05, 0.4, 0.05], [0.15, 0.65, 0.2], [0.4, 0.2, 0.4]]), # Meter 1 ] # Class 0 is the correct class for sample 1, class 2 for sample 2, etc targets = [ torch.tensor([[0, 1, 0], [1, 0, 0], [1, 1, 1]]), # Meter 0 torch.tensor([[0, 1, 0], [0, 1, 0], [0, 1, 0]]), # Meter 1 torch.tensor([[0, 1, 0], [1, 0, 0], [1, 1, 1]]), # Meter 0 torch.tensor([[0, 1, 0], [0, 1, 0], [0, 1, 0]]), # Meter 1 ] # In first two updates there are 4 correct top-1, 6 correct in top 2 # The same occurs in the second two updates and is added to first expected_values = [ { "top_1": 4 / 6.0, "top_2": 6 / 12.0 }, # After one update to each meter { "top_1": 8 / 12.0, "top_2": 12 / 24.0 }, # After two updates to each meter ] self.meter_distributed_test(meters, model_outputs, targets, expected_values)
def test_meter_invalid_target(self): meter = PrecisionAtKMeter(topk=[1, 2]) model_output = torch.tensor([ [0.2, 0.4, 0.4], # top-1: 1/2, top-2: 1/2 [0.2, 0.65, 0.15], # top-1: 1, top-2: 1/0 [0.33, 0.33, 0.34], # top-1: 2, top-2: 2/0/1 ]) # Target shape does not match model shape target = torch.tensor([0, 1, 2]) self.meter_invalid_meter_input_test(meter, model_output, target)
def __init__(self, num_meters: int, topk_values: List[int], meter_names: List[str]): super().__init__() assert is_pos_int(num_meters), "num_meters must be positive" assert isinstance(topk_values, list), "topk_values must be a list" assert len(topk_values) > 0, "topk_values list should have at least one element" assert [ is_pos_int(x) for x in topk_values ], "each value in topk_values must be >= 1" self._num_meters = num_meters self._topk_values = topk_values self._meters = [ PrecisionAtKMeter(self._topk_values) for _ in range(self._num_meters) ] self._meter_names = meter_names self.reset()
def test_meter_fp16(self): """ This test verifies that the meter works if the input tensor is fp16. """ meter = PrecisionAtKMeter(topk=[1, 2]) # Batchsize = 3, num classes = 3, score is probability of class model_output = torch.tensor([ [0.2, 0.4, 0.4], # top-1: 1/2, top-2: 1/2 [0.2, 0.65, 0.15], # top-1: 1, top-2: 1/0 [0.33, 0.33, 0.34], # top-1: 2, top-2: 2/0?1 ]).half() # One-hot encoding, 1 = positive for class # sample-1: 1, sample-2: 0, sample-3: 0,1,2 target = torch.tensor([[0, 1, 0], [1, 0, 0], [1, 1, 1]]).half() # Note for ties, we select randomly, so we should not use ambiguous ties expected_value = {"top_1": 2 / 3.0, "top_2": 4 / 6.0} self.meter_update_and_reset_test(meter, model_output, target, expected_value)