def test_compute_batch_images():
acc = Accuracy()
y_pred = torch.sigmoid(torch.rand(1, 2, 2))
y = torch.ones(1, 2, 2).type(torch.LongTensor)
y_pred = y_pred.unsqueeze(1)
indices = torch.max(torch.cat([1.0 - y_pred, y_pred], dim=1), dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.view(-1).data.numpy(), indices.view(-1).data.numpy()) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.sigmoid(torch.rand(2, 1, 2, 2))
y = torch.ones(2, 2, 2).type(torch.LongTensor)
indices = torch.max(torch.cat([1.0 - y_pred, y_pred], dim=1), dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.view(-1).data.numpy(), indices.view(-1).data.numpy()) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.sigmoid(torch.rand(2, 1, 2, 2))
y = torch.ones(2, 1, 2, 2).type(torch.LongTensor)
indices = torch.max(torch.cat([1.0 - y_pred, y_pred], dim=1), dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.view(-1).data.numpy(), indices.view(-1).data.numpy()) == pytest.approx(acc.compute())
def test_binary_input_NL():
# Binary accuracy on input of shape (N, L)
acc = Accuracy()
# TODO: y_pred should be binary after 0.1.2 release
# y_pred = torch.randint(0, 2, size=(10, 5)).type(torch.LongTensor)
y_pred = torch.rand(10, 5)
y = torch.randint(0, 2, size=(10, 5)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y = y.numpy().ravel()
# np_y_pred = y_pred.numpy().ravel()
np_y_pred = (y_pred.numpy().ravel() > 0.5).astype('int')
assert acc._type == 'binary'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
acc.reset()
# TODO: y_pred should be binary after 0.1.2 release
# y_pred = torch.randint(0, 2, size=(10, 1, 5)).type(torch.LongTensor)
y_pred = torch.rand(10, 1, 5)
y = torch.randint(0, 2, size=(10, 1, 5)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y = y.numpy().ravel()
# np_y_pred = y_pred.numpy().ravel()
np_y_pred = (y_pred.numpy().ravel() > 0.5).astype('int')
assert acc._type == 'binary'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def _test():
acc = Accuracy()
y_pred = torch.randint(0, 2, size=(10,)).long()
y = torch.randint(0, 2, size=(10,)).long()
acc.update((y_pred, y))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert acc._type == "binary"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
# Batched Updates
acc.reset()
y_pred = torch.randint(0, 2, size=(100,)).long()
y = torch.randint(0, 2, size=(100,)).long()
n_iters = 16
batch_size = y.shape[0] // n_iters + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size]))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert acc._type == "binary"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def test_no_update():
acc = Accuracy()
with pytest.raises(
NotComputableError,
match=
r"Accuracy must have at least one example before it can be computed"
):
acc.compute()
def save_confusion_matrix(data_root,
output_root,
segmenter,
data_subset="val"):
dataset = SegmentationDataset(data_root, data_subset)
confusion_matrix_caluclator = ConfusionMatrix(num_classes=2,
average="precision")
accuracy_calculator = Accuracy()
for image, mask_gt in dataset:
mask_pred = segmenter.get_raw_prediction(image)
mask_gt = torch.from_numpy(mask_gt).to(
mask_pred.device).unsqueeze(0).unsqueeze(0)
output = (mask_pred, mask_gt)
confusion_matrix_caluclator.update(
output_transform_confusion_matrix(output))
accuracy_calculator.update(output_transform_accuracy(output))
confusion_matrix = confusion_matrix_caluclator.compute()
accuracy = accuracy_calculator.compute()
cm_figure = plot_confusion_matrix(confusion_matrix)
filename_base = f"confusion_matrix_acc={accuracy:.6f}"
cm_figure.savefig(os.path.join(output_root, filename_base + ".pdf"))
cm_figure.savefig(os.path.join(output_root, filename_base + ".png"))
def test_categorical_compute():
acc = Accuracy()
y_pred = torch.softmax(torch.rand(4, 4), dim=1)
y = torch.ones(4).type(torch.LongTensor)
indices = torch.max(y_pred, dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.view(-1).data.numpy(), indices.view(-1).data.numpy()) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.softmax(torch.rand(2, 2), dim=1)
y = torch.ones(2).type(torch.LongTensor)
indices = torch.max(y_pred, dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.view(-1).data.numpy(), indices.view(-1).data.numpy()) == pytest.approx(acc.compute())
def eval_at_dev(model, dev_iter):
acc = Accuracy()
for sent, gd_label in dev_iter:
pred = model(sent.cuda(9)) #get model predict
acc.update((pred, gd_label.cuda(9)))
acc_rate = acc.compute()
print("current model over dev set accuracy rate: " + str(acc_rate))
return acc_rate
def test_binary_compute():
acc = Accuracy()
y_pred = torch.sigmoid(torch.rand(4, 1))
y = torch.ones(4).type(torch.LongTensor)
indices = torch.max(torch.cat([1.0 - y_pred, y_pred], dim=1), dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.data.numpy(), indices.data.numpy()) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.sigmoid(torch.rand(4))
y = torch.ones(4).type(torch.LongTensor)
y_pred = y_pred.unsqueeze(1)
indices = torch.max(torch.cat([1.0 - y_pred, y_pred], dim=1), dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.data.numpy(), indices.data.numpy()) == pytest.approx(acc.compute())
def test_multilabel_input(n_times, test_data_multilabel):
acc = Accuracy(is_multilabel=True)
y_pred, y, batch_size = test_data_multilabel
if batch_size > 1:
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
else:
acc.update((y_pred, y))
np_y_pred = to_numpy_multilabel(y_pred)
np_y = to_numpy_multilabel(y)
assert acc._type == "multilabel"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def test_binary_input(n_times, test_data_binary):
acc = Accuracy()
y_pred, y, batch_size = test_data_binary
acc.reset()
if batch_size > 1:
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
else:
acc.update((y_pred, y))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert acc._type == "binary"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def test_categorical_compute_batch_images():
acc = Accuracy()
y_pred = torch.softmax(torch.rand(2, 3, 2, 2), dim=1)
y = torch.LongTensor([[[0, 1],
[0, 1]],
[[0, 2],
[0, 2]]])
indices = torch.max(y_pred, dim=1)[1]
acc.update((y_pred, y))
assert isinstance(acc.compute(), float)
assert accuracy_score(y.view(-1).data.numpy(), indices.view(-1).data.numpy()) == pytest.approx(acc.compute())
def test_multiclass_input_N():
# Multiclass input data of shape (N, ) and (N, C)
acc = Accuracy()
y_pred = torch.rand(10, 4)
y = torch.randint(0, 4, size=(10,)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert acc._type == 'multiclass'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.rand(4, 10)
y = torch.randint(0, 10, size=(4, 1)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert acc._type == 'multiclass'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
# 2-classes
acc.reset()
y_pred = torch.rand(4, 2)
y = torch.randint(0, 2, size=(4, 1)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert acc._type == 'multiclass'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def evaluation(trainer, test_loader, device):
trainer.model.eval()
predictions = []
labels = []
metric = Accuracy()
with torch.no_grad():
for index, batch in enumerate(test_loader):
inputs = batch[0].to(device)
label = batch[1].to(device)
#import ipdb; ipdb.set_trace()
pred = trainer.model(inputs)[0]
metric.update((pred, label))
acc = metric.compute()
return acc
def evaluation(trainer, test_loader, device):
trainer.model.eval()
predictions = []
labels = []
metric = Accuracy()
with torch.no_grad():
for index, batch in enumerate(test_loader):
review = batch[0].to(device)
label = batch[1].to(device)
domain = batch[2].to(device)
length = batch[3].to(device)
pred, _ = trainer.model(review, length)
metric.update((pred, label))
acc = metric.compute()
return acc
def test(epoch, model, test_loader, writer, embeddings=None):
model.eval()
test_loss = 0
correct = 0
acc = Accuracy()
acc.reset()
all_targets = []
all_results = []
with torch.no_grad():
for data, targets in test_loader:
data, targets = data.to(device), targets.to(device)
# perform prediction
output = model(data)
test_loss += criterion(output, targets).item()
# Since during training sigmoid is applied in BCEWithLogitsLoss
# we also need to apply it here
output = torch.sigmoid(output)
# Make a hard decision threshold at 0.5
output[output > 0.5] = 1
output[output <= 0.5] = 0
acc.update((output, targets))
acc_value = acc.compute()
test_loss /= len(test_loader.sampler)
writer.add_scalar("Test Loss", test_loss, int((epoch + 1)))
writer.add_scalar("Test Acc", acc_value, int((epoch + 1)))
print(
"Test set: Average loss: {:.4f}, Accuracy: ({:.0f}%)\n".format(
test_loss, acc_value * 100
)
)
def _test():
acc = Accuracy(is_multilabel=True)
y_pred = torch.randint(0, 2, size=(4, 5, 12, 10))
y = torch.randint(0, 2, size=(4, 5, 12, 10)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y_pred = to_numpy_multilabel(
y_pred) # (N, C, H, W, ...) -> (N * H * W ..., C)
np_y = to_numpy_multilabel(
y) # (N, C, H, W, ...) -> (N * H * W ..., C)
assert acc._type == 'multilabel'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.randint(0, 2,
size=(4, 10, 12, 8)).type(torch.LongTensor)
y = torch.randint(0, 2, size=(4, 10, 12, 8)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y_pred = to_numpy_multilabel(
y_pred) # (N, C, H, W, ...) -> (N * H * W ..., C)
np_y = to_numpy_multilabel(
y) # (N, C, H, W, ...) -> (N * H * W ..., C)
assert acc._type == 'multilabel'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
# Batched Updates
acc.reset()
y_pred = torch.randint(0, 2, size=(100, 5, 12, 10))
y = torch.randint(0, 2, size=(100, 5, 12, 10)).type(torch.LongTensor)
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
np_y_pred = to_numpy_multilabel(
y_pred) # (N, C, L, ...) -> (N * L * ..., C)
np_y = to_numpy_multilabel(y) # (N, C, L, ...) -> (N * L ..., C)
assert acc._type == 'multilabel'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def _test():
acc = Accuracy()
y_pred = torch.randint(0, 2, size=(4, 1)).type(torch.LongTensor)
y = torch.randint(0, 2, size=(4, )).type(torch.LongTensor)
acc.update((y_pred, y))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert acc._type == 'binary'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.randint(0, 2, size=(4, 1, 12)).type(torch.LongTensor)
y = torch.randint(0, 2, size=(4, 12)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert acc._type == 'binary'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
# Batched Updates
acc.reset()
y_pred = torch.randint(0, 2,
size=(100, 1, 8, 8)).type(torch.LongTensor)
y = torch.randint(0, 2, size=(100, 8, 8)).type(torch.LongTensor)
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert acc._type == 'binary'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def _test():
acc = Accuracy()
y_pred = torch.rand(4, 5, 12, 10)
y = torch.randint(0, 5, size=(4, 12, 10)).long()
acc.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert acc._type == "multiclass"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.rand(4, 5, 10, 12, 8)
y = torch.randint(0, 5, size=(4, 10, 12, 8)).long()
acc.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert acc._type == "multiclass"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
# Batched Updates
acc.reset()
y_pred = torch.rand(100, 3, 8, 8)
y = torch.randint(0, 3, size=(100, 8, 8)).long()
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size]))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
assert acc._type == "multiclass"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
def _test():
acc = Accuracy(is_multilabel=True)
y_pred = torch.randint(0, 2, size=(10, 4))
y = torch.randint(0, 2, size=(10, 4)).long()
acc.update((y_pred, y))
np_y_pred = y_pred.numpy()
np_y = y.numpy()
assert acc._type == "multilabel"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.randint(0, 2, size=(50, 7)).long()
y = torch.randint(0, 2, size=(50, 7)).long()
acc.update((y_pred, y))
np_y_pred = y_pred.numpy()
np_y = y.numpy()
assert acc._type == "multilabel"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
# Batched Updates
acc.reset()
y_pred = torch.randint(0, 2, size=(100, 4))
y = torch.randint(0, 2, size=(100, 4)).long()
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size]))
np_y = y.numpy()
np_y_pred = y_pred.numpy()
assert acc._type == "multilabel"
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
shuffle=True,
num_workers=12)
accuracy = Accuracy()
best_acc = 0.0
for iter in range(args.num_iter):
accuracy.reset()
for batch in tqdm(dataloader):
input_ids = batch["input_ids"].to(device)
labels = batch["label"].to(device)
loss, logits, hidden_states, attentions = model(
input_ids, labels=labels).values()
optimizer.zero_grad()
loss.backward()
optimizer.step()
accuracy.update((logits, labels))
acc = accuracy.compute()
if acc > best_acc:
best_acc = acc
model.save_pretrained(
f"/data/model/hfl/chinese-macbert-base/{args.dataset}_{best_acc:.6f}"
)
print(f"Iter: {iter}, Acc: {acc:.4f}, Best Acc: {best_acc:.4f}")
def _test(metric_device):
metric_device = torch.device(metric_device)
acc = Accuracy(is_multilabel=True, device=metric_device)
torch.manual_seed(10 + rank)
y_pred = torch.randint(0, 2, size=(4, 5, 8, 10), device=device).long()
y = torch.randint(0, 2, size=(4, 5, 8, 10), device=device).long()
acc.update((y_pred, y))
assert (
acc._num_correct.device == metric_device
), f"{type(acc._num_correct.device)}:{acc._num_correct.device} vs {type(metric_device)}:{metric_device}"
# gather y_pred, y
y_pred = idist.all_gather(y_pred)
y = idist.all_gather(y)
np_y_pred = to_numpy_multilabel(y_pred.cpu()) # (N, C, H, W, ...) -> (N * H * W ..., C)
np_y = to_numpy_multilabel(y.cpu()) # (N, C, H, W, ...) -> (N * H * W ..., C)
assert acc._type == "multilabel"
n = acc._num_examples
res = acc.compute()
assert n * idist.get_world_size() == acc._num_examples
assert isinstance(res, float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(res)
acc.reset()
torch.manual_seed(10 + rank)
y_pred = torch.randint(0, 2, size=(4, 7, 10, 8), device=device).long()
y = torch.randint(0, 2, size=(4, 7, 10, 8), device=device).long()
acc.update((y_pred, y))
assert (
acc._num_correct.device == metric_device
), f"{type(acc._num_correct.device)}:{acc._num_correct.device} vs {type(metric_device)}:{metric_device}"
# gather y_pred, y
y_pred = idist.all_gather(y_pred)
y = idist.all_gather(y)
np_y_pred = to_numpy_multilabel(y_pred.cpu()) # (N, C, H, W, ...) -> (N * H * W ..., C)
np_y = to_numpy_multilabel(y.cpu()) # (N, C, H, W, ...) -> (N * H * W ..., C)
assert acc._type == "multilabel"
n = acc._num_examples
res = acc.compute()
assert n * idist.get_world_size() == acc._num_examples
assert isinstance(res, float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(res)
# check that result is not changed
res = acc.compute()
assert n * idist.get_world_size() == acc._num_examples
assert isinstance(res, float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(res)
# Batched Updates
acc.reset()
torch.manual_seed(10 + rank)
y_pred = torch.randint(0, 2, size=(80, 5, 8, 10), device=device).long()
y = torch.randint(0, 2, size=(80, 5, 8, 10), device=device).long()
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size]))
assert (
acc._num_correct.device == metric_device
), f"{type(acc._num_correct.device)}:{acc._num_correct.device} vs {type(metric_device)}:{metric_device}"
# gather y_pred, y
y_pred = idist.all_gather(y_pred)
y = idist.all_gather(y)
np_y_pred = to_numpy_multilabel(y_pred.cpu()) # (N, C, L, ...) -> (N * L * ..., C)
np_y = to_numpy_multilabel(y.cpu()) # (N, C, L, ...) -> (N * L ..., C)
assert acc._type == "multilabel"
n = acc._num_examples
res = acc.compute()
assert n * idist.get_world_size() == acc._num_examples
assert isinstance(res, float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(res)
def test_no_update():
acc = Accuracy()
with pytest.raises(NotComputableError):
acc.compute()
def _test():
acc = Accuracy()
y_pred = torch.rand(10, 4)
y = torch.randint(0, 4, size=(10, )).type(torch.LongTensor)
acc.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert acc._type == 'multiclass'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.rand(10, 10, 1)
y = torch.randint(0, 18, size=(10, 1)).type(torch.LongTensor)
acc.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert acc._type == 'multiclass'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.rand(10, 18)
y = torch.randint(0, 18, size=(10, )).type(torch.LongTensor)
acc.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert acc._type == 'multiclass'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
acc.reset()
y_pred = torch.rand(4, 10)
y = torch.randint(0, 10, size=(4, )).type(torch.LongTensor)
acc.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert acc._type == 'multiclass'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
# 2-classes
acc.reset()
y_pred = torch.rand(4, 2)
y = torch.randint(0, 2, size=(4, )).type(torch.LongTensor)
acc.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert acc._type == 'multiclass'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
# Batched Updates
acc.reset()
y_pred = torch.rand(100, 5)
y = torch.randint(0, 5, size=(100, )).type(torch.LongTensor)
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
acc.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
assert acc._type == 'multiclass'
assert isinstance(acc.compute(), float)
assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
class ClassificationTask(pl.LightningModule, TFLogger):
"""Standard interface for the trainer to interact with the model."""
def __init__(self, params):
super().__init__()
self.save_hyperparameters(params)
self.model = get_model(params)
self.loss = get_loss_fn(params)
self.val_acc = Accuracy()
def forward(self, x):
return self.model(x)
def training_step(self, batch, batch_nb):
"""
Returns:
A dictionary of loss and metrics, with:
loss(required): loss used to calculate the gradient
log: metrics to be logged to the TensorBoard and metrics.csv
progress_bar: metrics to be logged to the progress bar
and metrics.csv
"""
x, y = batch
logits = self.forward(x)
loss = self.loss(logits.view(-1), y)
return {'loss': loss, 'log': {'train_loss': loss}}
def validation_step(self, batch, batch_nb):
x, y = batch
logits = self.forward(x)
loss = self.loss(logits.view(-1), y)
y_hat = (logits > 0).float()
self.val_acc.update((y_hat, y))
return loss
def validation_epoch_end(self, outputs):
"""
Aggregate and return the validation metrics
Args:
outputs: A list of dictionaries of metrics from `validation_step()'
Returns: None
Returns:
A dictionary of loss and metrics, with:
val_loss (required): validation_loss
log: metrics to be logged to the TensorBoard and metrics.csv
progress_bar: metrics to be logged to the progress bar
and metrics.csv
"""
avg_loss = torch.stack(outputs).mean()
avg_acc = self.val_acc.compute()
self.val_acc.reset()
self.log("val_loss", avg_loss)
def test_step(self, batch, batch_nb):
x, y = batch
y_hat = self.forward(x)
loss = self.loss(y_hat.view(-1), y)
return {'test_loss': loss, 'log': {'test_loss': loss}}
def test_epoch_end(self, outputs):
avg_loss = torch.stack([x['test_loss'] for x in outputs]).mean()
return {'avg_test_loss': avg_loss}
def configure_optimizers(self):
return [torch.optim.Adam(self.parameters(), lr=0.02)]
def train_dataloader(self):
dataset = ImageClassificationDemoDataset()
return DataLoader(dataset, shuffle=True, batch_size=2, num_workers=8)
def val_dataloader(self):
dataset = ImageClassificationDemoDataset()
return DataLoader(dataset, shuffle=False, batch_size=1, num_workers=8)
def test_dataloader(self):
dataset = ImageClassificationDemoDataset()
return DataLoader(dataset, shuffle=False, batch_size=1, num_workers=8)
loss = Loss(F.cross_entropy)
precision = Precision()
sensitivity = Sensitivity()
specificity = Specificity()
for i in range(FG.fold):
parser.args.cur_fold = i
output, target = run_fold(parser, vis)
output = torch.cat(output)
target = torch.cat(target)
arg = (output, target)
acc.update(arg)
loss.update(arg)
precision.update(arg)
sensitivity.update(arg)
specificity.update(arg)
end = '<br>'
text = 'Over all result<br>'
text += 'accuracy: ' + '{:.4f}'.format(acc.compute()) + end
text += 'loss: ' + '{:.4f}'.format(loss.compute()) + end
text += 'precision: ' + '{}'.format(precision.compute()) + end
text += 'sensitivity: ' + '{}'.format(sensitivity.compute()) + end
text += 'specificity: ' + '{}'.format(specificity.compute()) + end
vis.text(text, 'result_overall')
vis.save([vis.env])
def test_zero_div():
acc = Accuracy()
with pytest.raises(NotComputableError):
acc.compute()
plt.subplot(122)
plt.plot(np.arange(len(train_loss)) + 1, train_loss, label='Train')
plt.plot(np.arange(len(validation_loss)) + 1, validation_loss,
label='Validation')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.grid()
plt.show()
# Test the model on the test images
images, masks = next(iter(agent.test_loader))
preds = agent.model(images)
acc = Accuracy()
acc.update([preds, masks])
print('Test Accuracy: {}'.format(acc.compute()))
loss = agent.criterion(preds, masks)
loss = loss.cpu().detach().numpy()
print('Test Loss: {}'.format(loss))
images = images.cpu().numpy()
masks = masks.cpu().numpy()
preds = torch.argmax(preds, dim=1).cpu().numpy()
plt.figure(figsize=(30, 10))
for i in range(len(images)):
plt.subplot(1, len(images), i + 1)
image = np.moveaxis(images[i], 0, -1)
image = image * 255
image = image.astype(int)
def test_ner_example():
acc = Accuracy()
y_pred = torch.softmax(torch.rand(2, 3, 8), dim=1)
y = torch.Tensor([[1, 1, 1, 1, 1, 1, 1, 1],
[2, 2, 2, 2, 2, 2, 2, 2]]).type(torch.LongTensor)
indices = torch.max(y_pred, dim=1)[1]
acc.update((y_pred, y))
assert accuracy_score(y.view(-1).data.numpy(), indices.view(-1).data.numpy()) == pytest.approx(acc.compute())
