def test_binary_input_NL():
# Binary accuracy on input of shape (N, L)
acc = Accuracy()
# TODO: y_pred should be binary after 0.1.2 release
# y_pred = torch.randint(0, 2, size=(10, 5)).type(torch.LongTensor)
y_pred = torch.rand(10, 5)
y = torch.randint(0, 2, size=(10, 5)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y = y.numpy().ravel()
# np_y_pred = y_pred.numpy().ravel()
np_y_pred = (y_pred.numpy().ravel() > 0.5).astype('int')
assert acc._type == 'binary'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
acc.reset()
# TODO: y_pred should be binary after 0.1.2 release
# y_pred = torch.randint(0, 2, size=(10, 1, 5)).type(torch.LongTensor)
y_pred = torch.rand(10, 1, 5)
y = torch.randint(0, 2, size=(10, 1, 5)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y = y.numpy().ravel()
# np_y_pred = y_pred.numpy().ravel()
np_y_pred = (y_pred.numpy().ravel() > 0.5).astype('int')
assert acc._type == 'binary'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def test_binary_wrong_inputs():
acc = Accuracy()
with pytest.raises(ValueError):
# y has not only 0 or 1 values
acc.update(
(torch.randint(0, 2, size=(10, )).long(), torch.arange(0,
10).long()))
with pytest.raises(ValueError):
# y_pred values are not thresholded to 0, 1 values
acc.update((torch.rand(10, ), torch.randint(0, 2, size=(10, )).long()))
with pytest.raises(ValueError):
# incompatible shapes
acc.update((torch.randint(0, 2, size=(10, )).long(),
torch.randint(0, 2, size=(10, 5)).long()))
with pytest.raises(ValueError):
# incompatible shapes
acc.update((torch.randint(0, 2, size=(10, 5, 6)).long(),
torch.randint(0, 2, size=(10, )).long()))
with pytest.raises(ValueError):
# incompatible shapes
acc.update((torch.randint(0, 2, size=(10, )).long(),
torch.randint(0, 2, size=(10, 5, 6)).long()))
def test_binary_wrong_inputs():
acc = Accuracy()
with pytest.raises(
ValueError,
match=r"For binary cases, y must be comprised of 0's and 1's"):
# y has not only 0 or 1 values
acc.update(
(torch.randint(0, 2, size=(10, )).long(), torch.arange(0,
10).long()))
with pytest.raises(
ValueError,
match=r"For binary cases, y_pred must be comprised of 0's and 1's"
):
# y_pred values are not thresholded to 0, 1 values
acc.update((
torch.rand(10, ),
torch.randint(0, 2, size=(10, )).long(),
))
with pytest.raises(ValueError, match=r"y must have shape of "):
# incompatible shapes
acc.update((torch.randint(0, 2, size=(10, )).long(),
torch.randint(0, 2, size=(10, 5)).long()))
with pytest.raises(ValueError, match=r"y must have shape of "):
# incompatible shapes
acc.update((torch.randint(0, 2, size=(10, 5, 6)).long(),
torch.randint(0, 2, size=(10, )).long()))
with pytest.raises(ValueError, match=r"y must have shape of "):
# incompatible shapes
acc.update((torch.randint(0, 2, size=(10, )).long(),
torch.randint(0, 2, size=(10, 5, 6)).long()))
def save_confusion_matrix(data_root,
output_root,
segmenter,
data_subset="val"):
dataset = SegmentationDataset(data_root, data_subset)
confusion_matrix_caluclator = ConfusionMatrix(num_classes=2,
average="precision")
accuracy_calculator = Accuracy()
for image, mask_gt in dataset:
mask_pred = segmenter.get_raw_prediction(image)
mask_gt = torch.from_numpy(mask_gt).to(
mask_pred.device).unsqueeze(0).unsqueeze(0)
output = (mask_pred, mask_gt)
confusion_matrix_caluclator.update(
output_transform_confusion_matrix(output))
accuracy_calculator.update(output_transform_accuracy(output))
confusion_matrix = confusion_matrix_caluclator.compute()
accuracy = accuracy_calculator.compute()
cm_figure = plot_confusion_matrix(confusion_matrix)
filename_base = f"confusion_matrix_acc={accuracy:.6f}"
cm_figure.savefig(os.path.join(output_root, filename_base + ".pdf"))
cm_figure.savefig(os.path.join(output_root, filename_base + ".png"))
def _test():
acc = Accuracy()
y_pred = torch.randint(0, 2, size=(10,)).long()
y = torch.randint(0, 2, size=(10,)).long()
acc.update((y_pred, y))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert acc._type == "binary"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
# Batched Updates
acc.reset()
y_pred = torch.randint(0, 2, size=(100,)).long()
y = torch.randint(0, 2, size=(100,)).long()
n_iters = 16
batch_size = y.shape[0] // n_iters + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size]))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert acc._type == "binary"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def eval_at_dev(model, dev_iter):
acc = Accuracy()
for sent, gd_label in dev_iter:
pred = model(sent.cuda(9)) #get model predict
acc.update((pred, gd_label.cuda(9)))
acc_rate = acc.compute()
print("current model over dev set accuracy rate: " + str(acc_rate))
return acc_rate
def test_ner_example():
acc = Accuracy()
y_pred = torch.softmax(torch.rand(2, 3, 8), dim=1)
y = torch.Tensor([[1, 1, 1, 1, 1, 1, 1, 1],
[2, 2, 2, 2, 2, 2, 2, 2]]).type(torch.LongTensor)
indices = torch.max(y_pred, dim=1)[1]
acc.update((y_pred, y))
assert accuracy_score(y.view(-1).data.numpy(), indices.view(-1).data.numpy()) == pytest.approx(acc.compute())
def test_categorical_compute_batch_images():
acc = Accuracy()
y_pred = torch.softmax(torch.rand(2, 3, 2, 2), dim=1)
y = torch.LongTensor([[[0, 1],
[0, 1]],
[[0, 2],
[0, 2]]])
indices = torch.max(y_pred, dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.view(-1).data.numpy(), indices.view(-1).data.numpy()) == pytest.approx(acc.compute())
def test_incorrect_shape():
acc = Accuracy()
y_pred = torch.zeros(2, 3, 2, 2)
y = torch.zeros(2, 3)
with pytest.raises(ValueError):
acc.update((y_pred, y))
y_pred = torch.zeros(2, 3, 2, 2)
y = torch.zeros(2, 3, 4, 4)
with pytest.raises(ValueError):
acc.update((y_pred, y))
def test_binary_wrong_inputs():
acc = Accuracy()
with pytest.raises(ValueError):
# y has not only 0 or 1 values
acc.update((torch.randint(0, 2, size=(10,)).type(torch.LongTensor),
torch.arange(0, 10).type(torch.LongTensor)))
# TODO: Uncomment the following after 0.1.2 release
# with pytest.raises(ValueError):
# # y_pred values are not thresholded to 0, 1 values
# acc.update((torch.rand(10, 1),
# torch.randint(0, 2, size=(10,)).type(torch.LongTensor)))
with pytest.raises(ValueError):
# incompatible shapes
acc.update((torch.randint(0, 2, size=(10,)).type(torch.LongTensor),
torch.randint(0, 2, size=(10, 5)).type(torch.LongTensor)))
with pytest.raises(ValueError):
# incompatible shapes
acc.update((torch.randint(0, 2, size=(10, 5, 6)).type(torch.LongTensor),
torch.randint(0, 2, size=(10,)).type(torch.LongTensor)))
with pytest.raises(ValueError):
# incompatible shapes
acc.update((torch.randint(0, 2, size=(10,)).type(torch.LongTensor),
torch.randint(0, 2, size=(10, 5, 6)).type(torch.LongTensor)))
def evaluation(trainer, test_loader, device):
trainer.model.eval()
predictions = []
labels = []
metric = Accuracy()
with torch.no_grad():
for index, batch in enumerate(test_loader):
inputs = batch[0].to(device)
label = batch[1].to(device)
#import ipdb; ipdb.set_trace()
pred = trainer.model(inputs)[0]
metric.update((pred, label))
acc = metric.compute()
return acc
def test_incorrect_type():
acc = Accuracy()
# Start as binary data
y_pred = torch.rand(4,)
y = torch.ones(4).type(torch.LongTensor)
acc.update((y_pred, y))
# And add a multiclass data
y_pred = torch.rand(4, 4)
y = torch.ones(4).type(torch.LongTensor)
with pytest.raises(RuntimeError):
acc.update((y_pred, y))
def test_incorrect_type():
acc = Accuracy()
# Start as binary data
y_pred = torch.randint(0, 2, size=(4,))
y = torch.ones(4).long()
acc.update((y_pred, y))
# And add a multiclass data
y_pred = torch.rand(4, 4)
y = torch.ones(4).long()
with pytest.raises(RuntimeError):
acc.update((y_pred, y))
def evaluation(trainer, test_loader, device):
trainer.model.eval()
predictions = []
labels = []
metric = Accuracy()
with torch.no_grad():
for index, batch in enumerate(test_loader):
review = batch[0].to(device)
label = batch[1].to(device)
domain = batch[2].to(device)
length = batch[3].to(device)
pred, _ = trainer.model(review, length)
metric.update((pred, label))
acc = metric.compute()
return acc
def test_categorical_compute():
acc = Accuracy()
y_pred = torch.softmax(torch.rand(4, 4), dim=1)
y = torch.ones(4).type(torch.LongTensor)
indices = torch.max(y_pred, dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.view(-1).data.numpy(), indices.view(-1).data.numpy()) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.softmax(torch.rand(2, 2), dim=1)
y = torch.ones(2).type(torch.LongTensor)
indices = torch.max(y_pred, dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.view(-1).data.numpy(), indices.view(-1).data.numpy()) == pytest.approx(acc.compute())
def test_multilabel_input(n_times, test_data_multilabel):
acc = Accuracy(is_multilabel=True)
y_pred, y, batch_size = test_data_multilabel
if batch_size > 1:
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
else:
acc.update((y_pred, y))
np_y_pred = to_numpy_multilabel(y_pred)
np_y = to_numpy_multilabel(y)
assert acc._type == "multilabel"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def test_binary_compute():
acc = Accuracy()
y_pred = torch.sigmoid(torch.rand(4, 1))
y = torch.ones(4).type(torch.LongTensor)
indices = torch.max(torch.cat([1.0 - y_pred, y_pred], dim=1), dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.data.numpy(), indices.data.numpy()) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.sigmoid(torch.rand(4))
y = torch.ones(4).type(torch.LongTensor)
y_pred = y_pred.unsqueeze(1)
indices = torch.max(torch.cat([1.0 - y_pred, y_pred], dim=1), dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.data.numpy(), indices.data.numpy()) == pytest.approx(acc.compute())
def test_binary_input(n_times, test_data_binary):
acc = Accuracy()
y_pred, y, batch_size = test_data_binary
acc.reset()
if batch_size > 1:
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
else:
acc.update((y_pred, y))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert acc._type == "binary"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def _test_distrib_accumulator_device(device):
metric_devices = [torch.device("cpu")]
if device.type != "xla":
metric_devices.append(idist.device())
for metric_device in metric_devices:
acc = Accuracy(device=metric_device)
assert acc._device == metric_device
assert (
acc._num_correct.device == metric_device
), f"{type(acc._num_correct.device)}:{acc._num_correct.device} vs {type(metric_device)}:{metric_device}"
y_pred = torch.randint(0, 2, size=(10,), device=device, dtype=torch.long)
y = torch.randint(0, 2, size=(10,), device=device, dtype=torch.long)
acc.update((y_pred, y))
assert (
acc._num_correct.device == metric_device
), f"{type(acc._num_correct.device)}:{acc._num_correct.device} vs {type(metric_device)}:{metric_device}"
def test_compute_batch_images():
acc = Accuracy()
y_pred = torch.sigmoid(torch.rand(1, 2, 2))
y = torch.ones(1, 2, 2).type(torch.LongTensor)
y_pred = y_pred.unsqueeze(1)
indices = torch.max(torch.cat([1.0 - y_pred, y_pred], dim=1), dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.view(-1).data.numpy(), indices.view(-1).data.numpy()) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.sigmoid(torch.rand(2, 1, 2, 2))
y = torch.ones(2, 2, 2).type(torch.LongTensor)
indices = torch.max(torch.cat([1.0 - y_pred, y_pred], dim=1), dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.view(-1).data.numpy(), indices.view(-1).data.numpy()) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.sigmoid(torch.rand(2, 1, 2, 2))
y = torch.ones(2, 1, 2, 2).type(torch.LongTensor)
indices = torch.max(torch.cat([1.0 - y_pred, y_pred], dim=1), dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.view(-1).data.numpy(), indices.view(-1).data.numpy()) == pytest.approx(acc.compute())
def test_multiclass_input_N():
# Multiclass input data of shape (N, ) and (N, C)
acc = Accuracy()
y_pred = torch.rand(10, 4)
y = torch.randint(0, 4, size=(10,)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert acc._type == 'multiclass'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.rand(4, 10)
y = torch.randint(0, 10, size=(4, 1)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert acc._type == 'multiclass'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
# 2-classes
acc.reset()
y_pred = torch.rand(4, 2)
y = torch.randint(0, 2, size=(4, 1)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert acc._type == 'multiclass'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def test(epoch, model, test_loader, writer, embeddings=None):
model.eval()
test_loss = 0
correct = 0
acc = Accuracy()
acc.reset()
all_targets = []
all_results = []
with torch.no_grad():
for data, targets in test_loader:
data, targets = data.to(device), targets.to(device)
# perform prediction
output = model(data)
test_loss += criterion(output, targets).item()
# Since during training sigmoid is applied in BCEWithLogitsLoss
# we also need to apply it here
output = torch.sigmoid(output)
# Make a hard decision threshold at 0.5
output[output > 0.5] = 1
output[output <= 0.5] = 0
acc.update((output, targets))
acc_value = acc.compute()
test_loss /= len(test_loader.sampler)
writer.add_scalar("Test Loss", test_loss, int((epoch + 1)))
writer.add_scalar("Test Acc", acc_value, int((epoch + 1)))
print(
"Test set: Average loss: {:.4f}, Accuracy: ({:.0f}%)\n".format(
test_loss, acc_value * 100
)
)
def test_multilabel_wrong_inputs():
acc = Accuracy(is_multilabel=True)
with pytest.raises(ValueError):
# incompatible shapes
acc.update((torch.randint(0, 2, size=(10,)), torch.randint(0, 2, size=(10,)).long()))
with pytest.raises(ValueError):
# incompatible y_pred
acc.update((torch.rand(10, 5), torch.randint(0, 2, size=(10, 5)).long()))
with pytest.raises(ValueError):
# incompatible y
acc.update((torch.randint(0, 5, size=(10, 5, 6)), torch.rand(10)))
with pytest.raises(ValueError):
# incompatible binary shapes
acc.update((torch.randint(0, 2, size=(10, 1)), torch.randint(0, 2, size=(10, 1)).long()))
def test_multiclass_wrong_inputs():
acc = Accuracy()
with pytest.raises(ValueError):
# incompatible shapes
acc.update((torch.rand(10, 5, 4), torch.randint(0, 2, size=(10,)).long()))
with pytest.raises(ValueError):
# incompatible shapes
acc.update((torch.rand(10, 5, 6), torch.randint(0, 5, size=(10, 5)).long()))
with pytest.raises(ValueError):
# incompatible shapes
acc.update((torch.rand(10), torch.randint(0, 5, size=(10, 5, 6)).long()))
def _test():
acc = Accuracy(is_multilabel=True)
y_pred = torch.randint(0, 2, size=(4, 5, 12, 10))
y = torch.randint(0, 2, size=(4, 5, 12, 10)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y_pred = to_numpy_multilabel(
y_pred) # (N, C, H, W, ...) -> (N * H * W ..., C)
np_y = to_numpy_multilabel(
y) # (N, C, H, W, ...) -> (N * H * W ..., C)
assert acc._type == 'multilabel'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.randint(0, 2,
size=(4, 10, 12, 8)).type(torch.LongTensor)
y = torch.randint(0, 2, size=(4, 10, 12, 8)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y_pred = to_numpy_multilabel(
y_pred) # (N, C, H, W, ...) -> (N * H * W ..., C)
np_y = to_numpy_multilabel(
y) # (N, C, H, W, ...) -> (N * H * W ..., C)
assert acc._type == 'multilabel'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
# Batched Updates
acc.reset()
y_pred = torch.randint(0, 2, size=(100, 5, 12, 10))
y = torch.randint(0, 2, size=(100, 5, 12, 10)).type(torch.LongTensor)
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
np_y_pred = to_numpy_multilabel(
y_pred) # (N, C, L, ...) -> (N * L * ..., C)
np_y = to_numpy_multilabel(y) # (N, C, L, ...) -> (N * L ..., C)
assert acc._type == 'multilabel'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def _test():
acc = Accuracy()
y_pred = torch.randint(0, 2, size=(4, 1)).type(torch.LongTensor)
y = torch.randint(0, 2, size=(4, )).type(torch.LongTensor)
acc.update((y_pred, y))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert acc._type == 'binary'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.randint(0, 2, size=(4, 1, 12)).type(torch.LongTensor)
y = torch.randint(0, 2, size=(4, 12)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert acc._type == 'binary'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
# Batched Updates
acc.reset()
y_pred = torch.randint(0, 2,
size=(100, 1, 8, 8)).type(torch.LongTensor)
y = torch.randint(0, 2, size=(100, 8, 8)).type(torch.LongTensor)
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert acc._type == 'binary'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def _test():
acc = Accuracy()
y_pred = torch.rand(4, 5, 12, 10)
y = torch.randint(0, 5, size=(4, 12, 10)).long()
acc.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert acc._type == "multiclass"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.rand(4, 5, 10, 12, 8)
y = torch.randint(0, 5, size=(4, 10, 12, 8)).long()
acc.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert acc._type == "multiclass"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
# Batched Updates
acc.reset()
y_pred = torch.rand(100, 3, 8, 8)
y = torch.randint(0, 3, size=(100, 8, 8)).long()
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size]))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
assert acc._type == "multiclass"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def _test():
acc = Accuracy(is_multilabel=True)
y_pred = torch.randint(0, 2, size=(10, 4))
y = torch.randint(0, 2, size=(10, 4)).long()
acc.update((y_pred, y))
np_y_pred = y_pred.numpy()
np_y = y.numpy()
assert acc._type == "multilabel"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.randint(0, 2, size=(50, 7)).long()
y = torch.randint(0, 2, size=(50, 7)).long()
acc.update((y_pred, y))
np_y_pred = y_pred.numpy()
np_y = y.numpy()
assert acc._type == "multilabel"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
# Batched Updates
acc.reset()
y_pred = torch.randint(0, 2, size=(100, 4))
y = torch.randint(0, 2, size=(100, 4)).long()
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size]))
np_y = y.numpy()
np_y_pred = y_pred.numpy()
assert acc._type == "multilabel"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
shuffle=True,
num_workers=12)
accuracy = Accuracy()
best_acc = 0.0
for iter in range(args.num_iter):
accuracy.reset()
for batch in tqdm(dataloader):
input_ids = batch["input_ids"].to(device)
labels = batch["label"].to(device)
loss, logits, hidden_states, attentions = model(
input_ids, labels=labels).values()
optimizer.zero_grad()
loss.backward()
optimizer.step()
accuracy.update((logits, labels))
acc = accuracy.compute()
if acc > best_acc:
best_acc = acc
model.save_pretrained(
f"/data/model/hfl/chinese-macbert-base/{args.dataset}_{best_acc:.6f}"
)
print(f"Iter: {iter}, Acc: {acc:.4f}, Best Acc: {best_acc:.4f}")
def _test(metric_device):
metric_device = torch.device(metric_device)
acc = Accuracy(is_multilabel=True, device=metric_device)
torch.manual_seed(10 + rank)
y_pred = torch.randint(0, 2, size=(4, 5, 8, 10), device=device).long()
y = torch.randint(0, 2, size=(4, 5, 8, 10), device=device).long()
acc.update((y_pred, y))
assert (
acc._num_correct.device == metric_device
), f"{type(acc._num_correct.device)}:{acc._num_correct.device} vs {type(metric_device)}:{metric_device}"
# gather y_pred, y
y_pred = idist.all_gather(y_pred)
y = idist.all_gather(y)
np_y_pred = to_numpy_multilabel(y_pred.cpu()) # (N, C, H, W, ...) -> (N * H * W ..., C)
np_y = to_numpy_multilabel(y.cpu()) # (N, C, H, W, ...) -> (N * H * W ..., C)
assert acc._type == "multilabel"
n = acc._num_examples
res = acc.compute()
assert n * idist.get_world_size() == acc._num_examples
assert isinstance(res, float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(res)
acc.reset()
torch.manual_seed(10 + rank)
y_pred = torch.randint(0, 2, size=(4, 7, 10, 8), device=device).long()
y = torch.randint(0, 2, size=(4, 7, 10, 8), device=device).long()
acc.update((y_pred, y))
assert (
acc._num_correct.device == metric_device
), f"{type(acc._num_correct.device)}:{acc._num_correct.device} vs {type(metric_device)}:{metric_device}"
# gather y_pred, y
y_pred = idist.all_gather(y_pred)
y = idist.all_gather(y)
np_y_pred = to_numpy_multilabel(y_pred.cpu()) # (N, C, H, W, ...) -> (N * H * W ..., C)
np_y = to_numpy_multilabel(y.cpu()) # (N, C, H, W, ...) -> (N * H * W ..., C)
assert acc._type == "multilabel"
n = acc._num_examples
res = acc.compute()
assert n * idist.get_world_size() == acc._num_examples
assert isinstance(res, float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(res)
# check that result is not changed
res = acc.compute()
assert n * idist.get_world_size() == acc._num_examples
assert isinstance(res, float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(res)
# Batched Updates
acc.reset()
torch.manual_seed(10 + rank)
y_pred = torch.randint(0, 2, size=(80, 5, 8, 10), device=device).long()
y = torch.randint(0, 2, size=(80, 5, 8, 10), device=device).long()
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size]))
assert (
acc._num_correct.device == metric_device
), f"{type(acc._num_correct.device)}:{acc._num_correct.device} vs {type(metric_device)}:{metric_device}"
# gather y_pred, y
y_pred = idist.all_gather(y_pred)
y = idist.all_gather(y)
np_y_pred = to_numpy_multilabel(y_pred.cpu()) # (N, C, L, ...) -> (N * L * ..., C)
np_y = to_numpy_multilabel(y.cpu()) # (N, C, L, ...) -> (N * L ..., C)
assert acc._type == "multilabel"
n = acc._num_examples
res = acc.compute()
assert n * idist.get_world_size() == acc._num_examples
assert isinstance(res, float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(res)
