def test_double_meter_update_and_reset(self):
meter = VideoAccuracyMeter(topk=[1, 2],
clips_per_video_train=1,
clips_per_video_test=2)
# Batchsize = 3, num classes = 3, clips_per_video is 2,
# score is a value in {1, 2, 3}.
# Data of two batch is provided
model_outputs = [
torch.tensor(
[[3, 2, 1], [3, 1, 2], [1, 2, 2], [1, 2, 3], [2, 2, 2],
[1, 3, 2]],
dtype=torch.float,
),
torch.tensor(
[[3, 2, 1], [3, 1, 2], [1, 2, 2], [1, 2, 3], [2, 2, 2],
[1, 3, 2]],
dtype=torch.float,
),
]
# Class 0 is the correct class for video 1, class 2 for video 2, and
# class 1 for video, in both batches
targets = [
torch.tensor([0, 0, 1, 1, 2, 2]),
torch.tensor([0, 0, 1, 1, 2, 2])
]
# First batch has top-1 accuracy of 1/3.0, top-2 accuracy of 2/3.0
# Second batch has top-1 accuracy of 2/3.0, top-2 accuracy of 3/3.0
expected_value = {"top_1": 2 / 6.0, "top_2": 6 / 6.0}
self.meter_update_and_reset_test(meter,
model_outputs,
targets,
expected_value,
is_train=False)
def test_single_meter_update_and_reset(self):
"""
This test verifies that the meter works as expected on a single
update + reset + same single update.
"""
meter = VideoAccuracyMeter(topk=[1, 2],
clips_per_video_train=1,
clips_per_video_test=2)
# Batchsize = 3, num classes = 3, clips_per_video is 2,
# score is a value in {1, 2, 3}
model_output = torch.tensor(
[[3, 2, 1], [3, 1, 2], [1, 2, 2], [1, 2, 3], [2, 2, 2], [1, 3, 2]],
dtype=torch.float,
)
# Class 0 is the correct class for video 1, class 2 for video 2, and
# class 1 for video
target = torch.tensor([0, 0, 1, 1, 2, 2])
# Only the first sample has top class correct, first and third
# sample have correct class in top 2
expected_value = {"top_1": 1 / 3.0, "top_2": 3 / 3.0}
self.meter_update_and_reset_test(meter,
model_output,
target,
expected_value,
is_train=False)
def test_meter_invalid_topk(self):
meter = VideoAccuracyMeter(topk=[1, 5],
clips_per_video_train=1,
clips_per_video_test=2)
model_output = torch.tensor(
[[3, 2, 1], [3, 1, 2], [1, 2, 2], [1, 2, 3], [2, 2, 2], [1, 3, 2]],
dtype=torch.float,
)
target = torch.tensor([0, 1, 2])
self.meter_invalid_meter_input_test(meter, model_output, target)
def test_meter_get_set_classy_state_test(self):
# In this test we update meter0 with model_output0 & target0
# and we update meter1 with model_output1 & target1 then
# transfer the state from meter1 to meter0 and validate they
# give same expected value.
# Expected value is the expected value of meter1
meters = [
VideoAccuracyMeter(topk=[1, 2],
clips_per_video_train=1,
clips_per_video_test=2),
VideoAccuracyMeter(topk=[1, 2],
clips_per_video_train=1,
clips_per_video_test=2),
]
# Batchsize = 3, num classes = 3, score is a value in {1, 2,
# 3}...3 is the highest score
model_outputs = [
torch.tensor(
[[1, 2, 3], [1, 1, 3], [2, 2, 1], [3, 2, 1], [2, 2, 2],
[2, 3, 1]],
dtype=torch.float,
),
torch.tensor(
[[3, 2, 1], [3, 1, 2], [1, 2, 2], [1, 2, 3], [2, 2, 2],
[1, 3, 2]],
dtype=torch.float,
),
]
# Class 2 is the correct class for sample 1, class 0 for sample 2, etc
targets = [
torch.tensor([0, 0, 1, 1, 2, 2]),
torch.tensor([0, 0, 1, 1, 2, 2])
]
# Value for second update
expected_value = {"top_1": 1 / 3.0, "top_2": 3 / 3.0}
self.meter_get_set_classy_state_test(meters,
model_outputs,
targets,
expected_value,
is_train=False)
def test_meter_invalid_model_output(self):
meter = VideoAccuracyMeter(topk=[1, 2],
clips_per_video_train=1,
clips_per_video_test=2)
# This model output has 3 dimensions instead of expected 2
model_output = torch.tensor(
[[[3, 2, 1], [1, 2, 3]], [[-1, -3, -4], [-10, -90, -100]]],
dtype=torch.float,
)
target = torch.tensor([0, 1, 2])
self.meter_invalid_meter_input_test(meter, model_output, target)
def test_meter_invalid_target(self):
meter = VideoAccuracyMeter(topk=[1, 2],
clips_per_video_train=1,
clips_per_video_test=2)
model_output = torch.tensor(
[[3, 2, 1], [3, 1, 2], [1, 2, 2], [1, 2, 3], [2, 2, 2], [1, 3, 2]],
dtype=torch.float,
)
# Target has 3 dimensions instead of expected 1 or 2
target = torch.tensor([[[0, 1, 2], [0, 1, 2]]])
self.meter_invalid_meter_input_test(meter, model_output, target)
# Target of clips from the same video is not consistent
target = torch.tensor([0, 2, 1, 1, 2, 2])
self.meter_invalid_update_test(meter,
model_output,
target,
is_train=False)
def test_meter_distributed(self):
# Meter0 will execute on one process, Meter1 on the other
meters = [
VideoAccuracyMeter(topk=[1, 2],
clips_per_video_train=1,
clips_per_video_test=2),
VideoAccuracyMeter(topk=[1, 2],
clips_per_video_train=1,
clips_per_video_test=2),
]
# Batchsize = 3, num classes = 3, score is a value in {1, 2,
# 3}...3 is the highest score
model_outputs = [
torch.tensor(
[[1, 2, 3], [1, 1, 3], [2, 2, 1], [3, 2, 1], [2, 2, 2],
[2, 3, 1]],
dtype=torch.float,
), # Meter 0
torch.tensor(
[[3, 2, 1], [3, 1, 2], [1, 2, 2], [1, 2, 3], [2, 2, 2],
[1, 3, 2]],
dtype=torch.float,
), # Meter 1
torch.tensor(
[[1, 2, 3], [1, 1, 3], [2, 2, 1], [3, 2, 1], [2, 2, 2],
[2, 3, 1]],
dtype=torch.float,
), # Meter 0
torch.tensor(
[[3, 2, 1], [3, 1, 2], [1, 2, 2], [1, 2, 3], [2, 2, 2],
[1, 3, 2]],
dtype=torch.float,
), # Meter 1
]
# For meter 0, class 2 is the correct class for sample 1, class 0 for sample 2,
# etc
targets = [
torch.tensor([0, 0, 1, 1, 2, 2]), # Meter 0
torch.tensor([0, 0, 1, 1, 2, 2]), # Meter 1
torch.tensor([0, 0, 1, 1, 2, 2]), # Meter 0
torch.tensor([0, 0, 1, 1, 2, 2]), # Meter 1
]
# In first two updates there are 3 correct top-2, 5 correct in top 2
# The same occurs in the second two updates and is added to first
expected_values = [
{
"top_1": 1 / 6.0,
"top_2": 4 / 6.0
}, # After one update to each meter
{
"top_1": 2 / 12.0,
"top_2": 8 / 12.0
}, # After two updates to each meter
]
self.meter_distributed_test(meters,
model_outputs,
targets,
expected_values,
is_train=False)
