def test_sequence_recall(self):
recall = UnigramRecall()
gold = torch.Tensor([[1, 2, 3], [2, 4, 8], [7, 1, 1]])
predictions = torch.Tensor([[[1, 2, 3], [1, 2, -1]],
[[2, 4, 8], [2, 5, 9]],
[[-1, -1, -1], [7, 1, -1]]])
recall(predictions, gold)
actual_recall = recall.get_metric()
numpy.testing.assert_almost_equal(actual_recall, 1)
def test_sequence_recall_accumulates_and_resets_correctly(self):
recall = UnigramRecall()
gold = torch.Tensor([[1, 2, 3]])
recall(torch.Tensor([[[1, 2, 3]]]), gold)
recall(torch.Tensor([[[7, 8, 4]]]), gold)
actual_recall = recall.get_metric(reset=True)
numpy.testing.assert_almost_equal(actual_recall, 1/2)
assert recall.correct_count == 0
assert recall.total_count == 0
def test_sequence_recall_accumulates_and_resets_correctly(self):
recall = UnigramRecall()
gold = torch.Tensor([[1, 2, 3]])
recall(torch.Tensor([[[1, 2, 3]]]), gold)
recall(torch.Tensor([[[7, 8, 4]]]), gold)
actual_recall = recall.get_metric(reset=True)
numpy.testing.assert_almost_equal(actual_recall, 1 / 2)
assert recall.correct_count == 0
assert recall.total_count == 0
def test_sequence_recall_accumulates_and_resets_correctly(self, device: str):
recall = UnigramRecall()
gold = torch.tensor([[1, 2, 3]], device=device)
recall(torch.tensor([[[1, 2, 3]]], device=device), gold)
recall(torch.tensor([[[7, 8, 4]]], device=device), gold)
actual_recall = recall.get_metric(reset=True)
assert_allclose(actual_recall, 1 / 2)
assert recall.correct_count == 0
assert recall.total_count == 0
def test_sequence_recall(self, device: str):
recall = UnigramRecall()
gold = torch.tensor([[1, 2, 3], [2, 4, 8], [7, 1, 1]], device=device)
predictions = torch.tensor(
[[[1, 2, 3], [1, 2, -1]], [[2, 4, 8], [2, 5, 9]], [[-1, -1, -1], [7, 1, -1]]],
device=device,
)
recall(predictions, gold)
actual_recall = recall.get_metric()
assert_allclose(actual_recall, 1)
def test_sequence_recall_respects_mask(self):
recall = UnigramRecall()
gold = torch.Tensor([[2, 4, 8], [1, 2, 3], [7, 1, 1], [11, 14, 17]])
predictions = torch.Tensor([
[[2, 4, 8], [2, 5, 9]], # 3/3
[[-1, 2, 4], [3, 8, -1]], # 2/2
[[-1, -1, -1], [7, 2, -1]], # 1/2
[[12, 13, 17], [11, 13, 18]] # 2/2
])
mask = torch.Tensor([[1, 1, 1], [0, 1, 1], [1, 1, 0], [1, 0, 1]])
recall(predictions, gold, mask)
actual_recall = recall.get_metric()
numpy.testing.assert_almost_equal(actual_recall, 7 / 8)
def __init__(self, num_classes: int, input_dim: int,
output_dim: int) -> None:
super().__init__()
self.embedder = Embedding(num_classes, input_dim)
self.decoder_cell = GRUCell(input_dim, output_dim)
self.output_projection_layer = Linear(output_dim, num_classes)
self.recall = UnigramRecall()
def test_sequence_recall(self):
recall = UnigramRecall()
gold = torch.Tensor([
[1, 2, 3],
[2, 4, 8],
[7, 1, 1]
])
predictions = torch.Tensor([
[[1, 2, 3], [1, 2, -1]],
[[2, 4, 8], [2, 5, 9]],
[[-1, -1, -1], [7, 1, -1]]
])
recall(predictions, gold)
actual_recall = recall.get_metric()
numpy.testing.assert_almost_equal(actual_recall, 1)
def test_distributed_accuracy(self):
gold = torch.tensor([[2, 4, 8], [1, 2, 3], [7, 1, 1], [11, 14, 17]])
predictions = torch.tensor([
[[2, 4, 8], [2, 5, 9]], # 3/3
[[-1, 2, 4], [3, 8, -1]], # 2/2
[[-1, -1, -1], [7, 2, -1]], # 1/2
[[12, 13, 17], [11, 13, 18]], # 2/2
])
mask = torch.tensor([[True, True, True], [False, True, True],
[True, True, False], [True, False, True]])
gold = [gold[:2], gold[2:]]
predictions = [predictions[:2], predictions[2:]]
mask = [mask[:2], mask[2:]]
metric_kwargs = {
"predictions": predictions,
"gold_labels": gold,
"mask": mask
}
desired_values = {"unigram_recall": 7 / 8}
run_distributed_test(
[-1, -1],
global_distributed_metric,
UnigramRecall(),
metric_kwargs,
desired_values,
exact=False,
)
def test_sequence_recall_respects_mask(self, device: str):
recall = UnigramRecall()
gold = torch.tensor([[2, 4, 8], [1, 2, 3], [7, 1, 1], [11, 14, 17]], device=device)
predictions = torch.tensor(
[
[[2, 4, 8], [2, 5, 9]], # 3/3
[[-1, 2, 4], [3, 8, -1]], # 2/2
[[-1, -1, -1], [7, 2, -1]], # 1/2
[[12, 13, 17], [11, 13, 18]], # 2/2
],
device=device,
)
mask = torch.tensor([[1, 1, 1], [0, 1, 1], [1, 1, 0], [1, 0, 1]], device=device)
recall(predictions, gold, mask)
actual_recall = recall.get_metric()
assert_allclose(actual_recall, 7 / 8)
def __init__(self, name: str, event2mind: Event2Mind, num_classes: int,
input_dim: int, output_dim: int) -> None:
self.embedder = Embedding(num_classes, input_dim)
event2mind.add_module(f"{name}_embedder", self.embedder)
self.decoder_cell = GRUCell(input_dim, output_dim)
event2mind.add_module(f"{name}_decoder_cell", self.decoder_cell)
self.output_projection_layer = Linear(output_dim, num_classes)
event2mind.add_module(f"{name}_output_project_layer",
self.output_projection_layer)
self.recall = UnigramRecall()
def test_sequence_recall_respects_mask(self):
recall = UnigramRecall()
gold = torch.Tensor([
[2, 4, 8],
[1, 2, 3],
[7, 1, 1],
[11, 14, 17]
])
predictions = torch.Tensor([
[[2, 4, 8], [2, 5, 9]], # 3/3
[[-1, 2, 4], [3, 8, -1]], # 2/2
[[-1, -1, -1], [7, 2, -1]], # 1/2
[[12, 13, 17], [11, 13, 18]] # 2/2
])
mask = torch.Tensor([
[1, 1, 1],
[0, 1, 1],
[1, 1, 0],
[1, 0, 1]
])
recall(predictions, gold, mask)
actual_recall = recall.get_metric()
numpy.testing.assert_almost_equal(actual_recall, 7/8)
def multiple_runs(
global_rank: int,
world_size: int,
gpu_id: Union[int, torch.device],
metric: UnigramRecall,
metric_kwargs: Dict[str, List[Any]],
desired_values: Dict[str, Any],
exact: Union[bool, Tuple[float, float]] = True,
):
kwargs = {}
# Use the arguments meant for the process with rank `global_rank`.
for argname in metric_kwargs:
kwargs[argname] = metric_kwargs[argname][global_rank]
for i in range(200):
metric(**kwargs)
assert desired_values["unigram_recall"] == metric.get_metric(
)["unigram_recall"]
def test_get_metric_on_new_object_works(self):
recall = UnigramRecall()
actual_recall = recall.get_metric(reset=True)
numpy.testing.assert_almost_equal(actual_recall, 0)
def test_get_metric_on_new_object_works(self, device: str):
recall = UnigramRecall()
actual_recall = recall.get_metric(reset=True)
assert_allclose(actual_recall, 0)
def test_get_metric_on_new_object_works(self):
recall = UnigramRecall()
actual_recall = recall.get_metric(reset=True)
numpy.testing.assert_almost_equal(actual_recall, 0)