def _test(average):
pr = Precision(average=average)
y_pred = torch.rand(10, 5, 18, 16)
y = torch.randint(0, 5, size=(10, 18, 16)).type(torch.LongTensor)
pr.update((y_pred, y))
num_classes = y_pred.shape[1]
np_y_pred = y_pred.argmax(dim=1).numpy().ravel()
np_y = y.numpy().ravel()
assert pr._type == 'multiclass'
assert isinstance(pr.compute(), float if average else torch.Tensor)
pr_compute = pr.compute() if average else pr.compute().numpy()
sk_average_parameter = 'macro' if average else None
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
sk_compute = precision_score(np_y, np_y_pred, labels=range(0, num_classes), average=sk_average_parameter)
assert sk_compute == pytest.approx(pr_compute)
pr.reset()
y_pred = torch.rand(10, 7, 20, 12)
y = torch.randint(0, 7, size=(10, 20, 12)).type(torch.LongTensor)
pr.update((y_pred, y))
num_classes = y_pred.shape[1]
np_y_pred = y_pred.argmax(dim=1).numpy().ravel()
np_y = y.numpy().ravel()
assert pr._type == 'multiclass'
assert isinstance(pr.compute(), float if average else torch.Tensor)
pr_compute = pr.compute() if average else pr.compute().numpy()
sk_average_parameter = 'macro' if average else None
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
sk_compute = precision_score(np_y, np_y_pred, labels=range(0, num_classes), average=sk_average_parameter)
assert sk_compute == pytest.approx(pr_compute)
# Batched Updates
pr.reset()
y_pred = torch.rand(100, 8, 12, 14)
y = torch.randint(0, 8, size=(100, 12, 14)).type(torch.LongTensor)
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
pr.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
num_classes = y_pred.shape[1]
np_y = y.numpy().ravel()
np_y_pred = y_pred.argmax(dim=1).numpy().ravel()
assert pr._type == 'multiclass'
assert isinstance(pr.compute(), float if average else torch.Tensor)
pr_compute = pr.compute() if average else pr.compute().numpy()
sk_average_parameter = 'macro' if average else None
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
sk_compute = precision_score(np_y, np_y_pred, labels=range(0, num_classes), average=sk_average_parameter)
assert sk_compute == pytest.approx(pr_compute)
def test_binary_wrong_inputs():
pr = Precision()
with pytest.raises(ValueError):
# y has not only 0 or 1 values
pr.update((torch.randint(0, 2,
size=(10, )).long(), torch.arange(0,
10).long()))
with pytest.raises(ValueError):
# y_pred values are not thresholded to 0, 1 values
pr.update((
torch.rand(10, ),
torch.randint(0, 2, size=(10, )).long(),
))
with pytest.raises(ValueError):
# incompatible shapes
pr.update((
torch.randint(0, 2, size=(10, )).long(),
torch.randint(0, 2, size=(10, 5)).long(),
))
with pytest.raises(ValueError):
# incompatible shapes
pr.update((
torch.randint(0, 2, size=(10, 5, 6)).long(),
torch.randint(0, 2, size=(10, )).long(),
))
with pytest.raises(ValueError):
# incompatible shapes
pr.update((
torch.randint(0, 2, size=(10, )).long(),
torch.randint(0, 2, size=(10, 5, 6)).long(),
))
def _test(y_pred, y, batch_size):
pr.reset()
if batch_size > 1:
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
pr.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size]))
else:
pr.update((y_pred, y))
np_y_pred = to_numpy_multilabel(y_pred)
np_y = to_numpy_multilabel(y)
assert pr._type == "multilabel"
pr_compute = pr.compute() if average else pr.compute().mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert precision_score(np_y, np_y_pred, average="samples") == pytest.approx(pr_compute)
pr1 = Precision(is_multilabel=True, average=True)
pr2 = Precision(is_multilabel=True, average=False)
pr1.update((y_pred, y))
pr2.update((y_pred, y))
assert pr1.compute() == pytest.approx(pr2.compute().mean().item())
def test_multilabel_wrong_inputs():
pr = Precision(average=True, is_multilabel=True)
with pytest.raises(ValueError):
# incompatible shapes
pr.update((torch.randint(0, 2, size=(10,)), torch.randint(0, 2, size=(10,)).long()))
with pytest.raises(ValueError):
# incompatible y_pred
pr.update((torch.rand(10, 5), torch.randint(0, 2, size=(10, 5)).long()))
with pytest.raises(ValueError):
# incompatible y
pr.update((torch.randint(0, 5, size=(10, 5, 6)), torch.rand(10)))
with pytest.raises(ValueError):
# incompatible shapes between two updates
pr.update((torch.randint(0, 2, size=(20, 5)), torch.randint(0, 2, size=(20, 5)).long()))
pr.update((torch.randint(0, 2, size=(20, 6)), torch.randint(0, 2, size=(20, 6)).long()))
def _test(average):
pr = Precision(average=average, is_multilabel=True)
y_pred = torch.randint(0, 2, size=(10, 5, 18, 16))
y = torch.randint(0, 2, size=(10, 5, 18, 16)).long()
pr.update((y_pred, y))
np_y_pred = to_numpy_multilabel(y_pred)
np_y = to_numpy_multilabel(y)
assert pr._type == "multilabel"
pr_compute = pr.compute() if average else pr.compute().mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert precision_score(
np_y, np_y_pred,
average="samples") == pytest.approx(pr_compute)
pr.reset()
y_pred = torch.randint(0, 2, size=(10, 4, 20, 23))
y = torch.randint(0, 2, size=(10, 4, 20, 23)).long()
pr.update((y_pred, y))
np_y_pred = to_numpy_multilabel(y_pred)
np_y = to_numpy_multilabel(y)
assert pr._type == "multilabel"
pr_compute = pr.compute() if average else pr.compute().mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert precision_score(
np_y, np_y_pred,
average="samples") == pytest.approx(pr_compute)
# Batched Updates
pr.reset()
y_pred = torch.randint(0, 2, size=(100, 5, 12, 14))
y = torch.randint(0, 2, size=(100, 5, 12, 14)).long()
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
pr.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
np_y = to_numpy_multilabel(y)
np_y_pred = to_numpy_multilabel(y_pred)
assert pr._type == "multilabel"
pr_compute = pr.compute() if average else pr.compute().mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert precision_score(
np_y, np_y_pred,
average="samples") == pytest.approx(pr_compute)
def _test(average):
pr = Precision(average=average, is_multilabel=True)
y_pred = torch.randint(0, 2, size=(20, 5))
y = torch.randint(0, 2, size=(20, 5)).type(torch.LongTensor)
pr.update((y_pred, y))
np_y_pred = y_pred.numpy()
np_y = y.numpy()
assert pr._type == 'multilabel'
pr_compute = pr.compute() if average else pr.compute().mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert precision_score(
np_y, np_y_pred,
average='samples') == pytest.approx(pr_compute)
pr.reset()
y_pred = torch.randint(0, 2, size=(10, 4))
y = torch.randint(0, 2, size=(10, 4)).type(torch.LongTensor)
pr.update((y_pred, y))
np_y_pred = y_pred.numpy()
np_y = y.numpy()
assert pr._type == 'multilabel'
pr_compute = pr.compute() if average else pr.compute().mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert precision_score(
np_y, np_y_pred,
average='samples') == pytest.approx(pr_compute)
# Batched Updates
pr.reset()
y_pred = torch.randint(0, 2, size=(100, 4))
y = torch.randint(0, 2, size=(100, 4)).type(torch.LongTensor)
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
pr.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
np_y = y.numpy()
np_y_pred = y_pred.numpy()
assert pr._type == 'multilabel'
pr_compute = pr.compute() if average else pr.compute().mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert precision_score(
np_y, np_y_pred,
average='samples') == pytest.approx(pr_compute)
def test_multiclass_wrong_inputs():
pr = Precision()
with pytest.raises(ValueError):
# incompatible shapes
pr.update((torch.rand(10, 5, 4), torch.randint(0, 2, size=(10,)).type(torch.LongTensor)))
with pytest.raises(ValueError):
# incompatible shapes
pr.update((torch.rand(10, 5, 6), torch.randint(0, 5, size=(10, 5)).type(torch.LongTensor)))
with pytest.raises(ValueError):
# incompatible shapes
pr.update((torch.rand(10), torch.randint(0, 5, size=(10, 5, 6)).type(torch.LongTensor)))
def _test(average):
pr = Precision(average=average)
y_pred = torch.rand(20, 6)
y = torch.randint(0, 5, size=(20, )).type(torch.LongTensor)
pr.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert pr._type == 'multiclass'
assert isinstance(pr.compute(), float if average else torch.Tensor)
pr_compute = pr.compute() if average else pr.compute().numpy()
sklearn_average_parameter = 'macro' if average else None
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert precision_score(
np_y, np_y_pred,
average=sklearn_average_parameter) == pytest.approx(pr_compute)
pr.reset()
y_pred = torch.rand(10, 4)
y = torch.randint(0, 3, size=(10, 1)).type(torch.LongTensor)
pr.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert pr._type == 'multiclass'
assert isinstance(pr.compute(), float if average else torch.Tensor)
pr_compute = pr.compute() if average else pr.compute().numpy()
sklearn_average_parameter = 'macro' if average else None
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert precision_score(
np_y, np_y_pred,
average=sklearn_average_parameter) == pytest.approx(pr_compute)
# 2-classes
pr.reset()
y_pred = torch.rand(10, 2)
y = torch.randint(0, 2, size=(10, 1)).type(torch.LongTensor)
pr.update((y_pred, y))
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert pr._type == 'multiclass'
assert isinstance(pr.compute(), float if average else torch.Tensor)
pr_compute = pr.compute() if average else pr.compute().numpy()
sklearn_average_parameter = 'macro' if average else None
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert precision_score(
np_y, np_y_pred,
average=sklearn_average_parameter) == pytest.approx(pr_compute)
def _test(average):
pr = Precision(average=average)
y_pred = torch.randint(0, 2, size=(10, 12, 10))
y = torch.randint(0, 2, size=(10, 12, 10)).type(torch.LongTensor)
pr.update((y_pred, y))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert pr._type == 'binary'
assert isinstance(pr.compute(), float if average else torch.Tensor)
pr_compute = pr.compute() if average else pr.compute().numpy()
assert precision_score(np_y, np_y_pred,
average='binary') == pytest.approx(pr_compute)
pr.reset()
y_pred = torch.randint(0, 2, size=(10, 1, 12, 10))
y = torch.randint(0, 2, size=(10, 1, 12, 10)).type(torch.LongTensor)
pr.update((y_pred, y))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert pr._type == 'binary'
assert isinstance(pr.compute(), float if average else torch.Tensor)
pr_compute = pr.compute() if average else pr.compute().numpy()
assert precision_score(np_y, np_y_pred,
average='binary') == pytest.approx(pr_compute)
pr = Precision(average=average)
# Batched Updates
pr.reset()
y_pred = torch.randint(0, 2, size=(100, 12, 10))
y = torch.randint(0, 2, size=(100, 1, 12, 10)).type(torch.LongTensor)
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
pr.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert pr._type == 'binary'
assert isinstance(pr.compute(), float if average else torch.Tensor)
pr_compute = pr.compute() if average else pr.compute().numpy()
assert precision_score(np_y, np_y_pred,
average='binary') == pytest.approx(pr_compute)
def _test(average):
pr = Precision(average=average)
y_pred = torch.rand(10, 1)
y = torch.randint(0, 2, size=(10, )).type(torch.LongTensor)
pr.update((y_pred, y))
np_y = y.numpy().ravel()
# np_y_pred = y_pred.numpy().ravel()
np_y_pred = (y_pred.numpy().ravel() > 0.5).astype('int')
assert pr._type == 'binary'
assert isinstance(pr.compute(), float if average else torch.Tensor)
pr_compute = pr.compute() if average else pr.compute().numpy()
assert precision_score(np_y, np_y_pred,
average='binary') == pytest.approx(pr_compute)
pr.reset()
# TODO: y_pred should be binary after 0.1.2 release
# y_pred = torch.randint(0, 2, size=(10, )).type(torch.LongTensor)
y_pred = torch.rand(10)
y = torch.randint(0, 2, size=(10, )).type(torch.LongTensor)
pr.update((y_pred, y))
np_y = y.numpy().ravel()
# np_y_pred = y_pred.numpy().ravel()
np_y_pred = (y_pred.numpy().ravel() > 0.5).astype('int')
assert pr._type == 'binary'
assert isinstance(pr.compute(), float if average else torch.Tensor)
pr_compute = pr.compute() if average else pr.compute().numpy()
assert precision_score(np_y, np_y_pred,
average='binary') == pytest.approx(pr_compute)
pr.reset()
# TODO: y_pred should be binary after 0.1.2 release
y_pred = torch.Tensor(
[0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.51])
y = torch.randint(0, 2, size=(10, )).type(torch.LongTensor)
pr.update((y_pred, y))
np_y = y.numpy().ravel()
# np_y_pred = y_pred.numpy().ravel()
np_y_pred = (y_pred.numpy().ravel() > 0.5).astype('int')
assert pr._type == 'binary'
assert isinstance(pr.compute(), float if average else torch.Tensor)
pr_compute = pr.compute() if average else pr.compute().numpy()
assert precision_score(np_y, np_y_pred,
average='binary') == pytest.approx(pr_compute)
def test_compute():
precision = Precision()
y_pred = torch.eye(4)
y = torch.ones(4).type(torch.LongTensor)
precision.update((y_pred, y))
results = list(precision.compute())
assert results[0] == 0.0
assert results[1] == 1.0
assert results[2] == 0.0
assert results[3] == 0.0
precision.reset()
y_pred = torch.eye(2)
y = torch.ones(2).type(torch.LongTensor)
precision.update((y_pred, y))
y = torch.zeros(2).type(torch.LongTensor)
precision.update((y_pred, y))
results = list(precision.compute())
assert results[0] == 0.5
assert results[1] == 0.5
def test_multiclass_wrong_inputs():
pr = Precision()
with pytest.raises(ValueError):
# incompatible shapes
pr.update((torch.rand(10, 5, 4), torch.randint(0, 2, size=(10,)).long()))
with pytest.raises(ValueError):
# incompatible shapes
pr.update((torch.rand(10, 5, 6), torch.randint(0, 5, size=(10, 5)).long()))
with pytest.raises(ValueError):
# incompatible shapes
pr.update((torch.rand(10), torch.randint(0, 5, size=(10, 5, 6)).long()))
pr = Precision(average=True)
with pytest.raises(ValueError):
# incompatible shapes between two updates
pr.update((torch.rand(10, 5), torch.randint(0, 5, size=(10,)).long()))
pr.update((torch.rand(10, 6), torch.randint(0, 5, size=(10,)).long()))
with pytest.raises(ValueError):
# incompatible shapes between two updates
pr.update((torch.rand(10, 5, 12, 14), torch.randint(0, 5, size=(10, 12, 14)).long()))
pr.update((torch.rand(10, 6, 12, 14), torch.randint(0, 5, size=(10, 12, 14)).long()))
pr = Precision(average=False)
with pytest.raises(ValueError):
# incompatible shapes between two updates
pr.update((torch.rand(10, 5), torch.randint(0, 5, size=(10,)).long()))
pr.update((torch.rand(10, 6), torch.randint(0, 5, size=(10,)).long()))
with pytest.raises(ValueError):
# incompatible shapes between two updates
pr.update((torch.rand(10, 5, 12, 14), torch.randint(0, 5, size=(10, 12, 14)).long()))
pr.update((torch.rand(10, 6, 12, 14), torch.randint(0, 5, size=(10, 12, 14)).long()))
def _test_distrib_integration_multilabel(device):
from ignite.engine import Engine
rank = idist.get_rank()
torch.manual_seed(12)
def _test(average, n_epochs, metric_device):
n_iters = 60
s = 16
n_classes = 7
offset = n_iters * s
y_true = torch.randint(0,
2,
size=(offset * idist.get_world_size(),
n_classes, 6, 8)).to(device)
y_preds = torch.randint(0,
2,
size=(offset * idist.get_world_size(),
n_classes, 6, 8)).to(device)
def update(engine, i):
return (
y_preds[i * s + rank * offset:(i + 1) * s + rank * offset,
...],
y_true[i * s + rank * offset:(i + 1) * s + rank * offset, ...],
)
engine = Engine(update)
pr = Precision(average=average,
is_multilabel=True,
device=metric_device)
pr.attach(engine, "pr")
data = list(range(n_iters))
engine.run(data=data, max_epochs=n_epochs)
assert "pr" in engine.state.metrics
res = engine.state.metrics["pr"]
res2 = pr.compute()
if isinstance(res, torch.Tensor):
res = res.cpu().numpy()
res2 = res2.cpu().numpy()
assert (res == res2).all()
else:
assert res == res2
np_y_preds = to_numpy_multilabel(y_preds)
np_y_true = to_numpy_multilabel(y_true)
assert pr._type == "multilabel"
res = res if average else res.mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert precision_score(np_y_true, np_y_preds,
average="samples") == pytest.approx(res)
metric_devices = ["cpu"]
if device.type != "xla":
metric_devices.append(idist.device())
for _ in range(2):
for metric_device in metric_devices:
_test(average=True, n_epochs=1, metric_device=metric_device)
_test(average=True, n_epochs=2, metric_device=metric_device)
_test(average=False, n_epochs=1, metric_device=metric_device)
_test(average=False, n_epochs=2, metric_device=metric_device)
pr1 = Precision(is_multilabel=True, average=True)
pr2 = Precision(is_multilabel=True, average=False)
y_pred = torch.randint(0, 2, size=(10, 4, 20, 23))
y = torch.randint(0, 2, size=(10, 4, 20, 23)).long()
pr1.update((y_pred, y))
pr2.update((y_pred, y))
assert pr1.compute() == pytest.approx(pr2.compute().mean().item())
def test(model, test_loader, experiment, hyperparams):
"""
Validates the model performance as LM on never-seen data.
:param model: the trained model to use for prediction
:param test_loader: Dataloader of testing data
:param experiment: comet.ml experiment object
:param hyperparams: hyperparameters dictionary
:param bpe: is bpe dataset or not
"""
# TODO: Define loss function, total loss, and total word count
loss_fn = nn.CrossEntropyLoss(ignore_index=0, reduction="sum")
total_loss = 0
word_count = 0
total_correct = 0
total_words = 0
precision = Precision(average=False,
is_multilabel=True) # , device = device)
recall = Recall(average=False, is_multilabel=True) # , device = device)
model = model.eval()
with experiment.test():
# TODO: Write testing loop
with torch.no_grad():
for batch in tqdm(test_loader):
inputs = batch['lm_input']
labels = batch['lm_label']
word_count = sum([len(arr) for arr in inputs])
inputs = inputs.to(device)
labels = labels.to(device)
y_pred = model(inputs)
loss = loss_fn(torch.transpose(y_pred, 1, 2), labels)
total_loss += loss
prbs = torch.nn.functional.softmax(y_pred, dim=2)
batch_acc, batch_correct, batch_words = accuracy_func(
prbs, labels, word_count)
total_correct += batch_correct
total_words += batch_words
predicted_ids = torch.argmax(prbs, dim=2)
y_pred_one_hot = F.one_hot(predicted_ids.to(torch.int64),
num_classes=prbs.shape[-1])
labels_one_hot = F.one_hot(labels.to(torch.int64),
num_classes=prbs.shape[-1])
precision.update((torch.transpose(y_pred_one_hot, 1, 2),
torch.transpose(labels_one_hot, 1, 2)))
recall.update((torch.transpose(y_pred_one_hot, 1, 2),
torch.transpose(labels_one_hot, 1, 2)))
perplexity = torch.exp(total_loss / total_words)
accuracy = total_correct / total_words
perplexity = perplexity.item()
# accuracy = accuracy.cpu()
F1 = (precision * recall * 2 /
(precision + recall)).mean().compute().item()
print("F1: ", F1)
experiment.log_metric("F1", F1)
print("total_correct: ", total_correct)
print("total_words: ", total_words)
print("perplexity:", perplexity)
print("accuracy:", accuracy)
experiment.log_metric("perplexity", perplexity)
experiment.log_metric("accuracy", accuracy)
loss = Loss(F.cross_entropy)
precision = Precision()
sensitivity = Sensitivity()
specificity = Specificity()
for i in range(FG.fold):
parser.args.cur_fold = i
output, target = run_fold(parser, vis)
output = torch.cat(output)
target = torch.cat(target)
arg = (output, target)
acc.update(arg)
loss.update(arg)
precision.update(arg)
sensitivity.update(arg)
specificity.update(arg)
end = '<br>'
text = 'Over all result<br>'
text += 'accuracy: ' + '{:.4f}'.format(acc.compute()) + end
text += 'loss: ' + '{:.4f}'.format(loss.compute()) + end
text += 'precision: ' + '{}'.format(precision.compute()) + end
text += 'sensitivity: ' + '{}'.format(sensitivity.compute()) + end
text += 'specificity: ' + '{}'.format(specificity.compute()) + end
vis.text(text, 'result_overall')
vis.save([vis.env])
def train_predict(dataloader_train,dataloader_val,model,epochs,learning_rate,use_cuda):
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
if use_cuda:
model = model.cuda()
model = model.train()
start.record()
train_loss_list=[]
val_loss_list=[]
train_f1=[]
val_f1=[]
loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
precision = Precision()
recall = Recall()
f1 = Fbeta(beta=1.0, average=True, precision=precision, recall=recall)
for epoch in range(epochs):
print("Epoch: {}".format(epoch+1))
for i,(img, label) in enumerate(dataloader_train):
img, label = Variable(img),Variable(label)
if use_cuda:
img = img.cuda()
label = label.cuda()
optimizer.zero_grad()
pred = model.forward(img)
_,my_label = torch.max(label, dim=1)
loss = loss_fn(pred,my_label)
if i == len(dataloader_train)-1:
train_loss_list.append(loss.item())
loss.backward()
optimizer.step()
precision.update((pred, my_label))
recall.update((pred, my_label))
f1.update((pred, my_label))
print("\tTrain loss: {:0.2f}".format(train_loss_list[-1]))
precision.compute()
recall.compute()
train_f1.append(f1.compute()*100)
print("\tTrain F1 Score: {:0.2f}%".format(train_f1[-1]))
precision = Precision()
recall = Recall()
f1 = Fbeta(beta=1.0, average=True, precision=precision, recall=recall)
with torch.no_grad():
for i,(img, label) in enumerate(dataloader_val):
img, labels = Variable(img),Variable(label)
if use_cuda:
img = img.cuda()
label = label.cuda()
pred = model(img)
_,my_label = torch.max(label, dim=1)
loss = loss_fn(pred,my_label)
if i == len(dataloader_val)-1:
val_loss_list.append(loss.item())
precision.update((pred, my_label))
recall.update((pred, my_label))
f1.update((pred, my_label))
print("\n\tVal loss: {:0.2f}".format(val_loss_list[-1]))
precision.compute()
recall.compute()
val_f1.append(f1.compute()*100)
print("\tVal F1 Score: {:0.2f}%".format(val_f1[-1]))
end.record()
torch.cuda.synchronize()
time = start.elapsed_time(end)
return (train_loss_list,val_loss_list,train_f1,val_f1,time,model)
def _test_distrib_integration_multilabel(device):
from ignite.engine import Engine
rank = idist.get_rank()
torch.manual_seed(12)
def _test(average, n_epochs):
n_iters = 60
s = 16
n_classes = 7
offset = n_iters * s
y_true = torch.randint(0, 2, size=(offset * idist.get_world_size(), n_classes, 6, 8)).to(device)
y_preds = torch.randint(0, 2, size=(offset * idist.get_world_size(), n_classes, 6, 8)).to(device)
def update(engine, i):
return (
y_preds[i * s + rank * offset : (i + 1) * s + rank * offset, ...],
y_true[i * s + rank * offset : (i + 1) * s + rank * offset, ...],
)
engine = Engine(update)
pr = Precision(average=average, is_multilabel=True)
pr.attach(engine, "pr")
data = list(range(n_iters))
engine.run(data=data, max_epochs=n_epochs)
assert "pr" in engine.state.metrics
res = engine.state.metrics["pr"]
res2 = pr.compute()
if isinstance(res, torch.Tensor):
res = res.cpu().numpy()
res2 = res2.cpu().numpy()
assert (res == res2).all()
else:
assert res == res2
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
true_res = precision_score(
to_numpy_multilabel(y_true), to_numpy_multilabel(y_preds), average="samples" if average else None
)
assert pytest.approx(res) == true_res
for _ in range(2):
_test(average=True, n_epochs=1)
_test(average=True, n_epochs=2)
if idist.get_world_size() > 1:
with pytest.warns(
RuntimeWarning,
match="Precision/Recall metrics do not work in distributed setting when "
"average=False and is_multilabel=True",
):
pr = Precision(average=False, is_multilabel=True)
y_pred = torch.randint(0, 2, size=(4, 3, 6, 8))
y = torch.randint(0, 2, size=(4, 3, 6, 8)).long()
pr.update((y_pred, y))
pr_compute1 = pr.compute()
pr_compute2 = pr.compute()
assert len(pr_compute1) == 4 * 6 * 8
assert (pr_compute1 == pr_compute2).all()
def test_multilabel_input_NCHW():
def _test(average):
pr = Precision(average=average, is_multilabel=True)
y_pred = torch.randint(0, 2, size=(10, 5, 18, 16))
y = torch.randint(0, 2, size=(10, 5, 18, 16)).type(torch.LongTensor)
pr.update((y_pred, y))
np_y_pred = to_numpy_multilabel(y_pred)
np_y = to_numpy_multilabel(y)
assert pr._type == 'multilabel'
pr_compute = pr.compute() if average else pr.compute().mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert precision_score(
np_y, np_y_pred,
average='samples') == pytest.approx(pr_compute)
pr.reset()
y_pred = torch.randint(0, 2, size=(10, 4, 20, 23))
y = torch.randint(0, 2, size=(10, 4, 20, 23)).type(torch.LongTensor)
pr.update((y_pred, y))
np_y_pred = to_numpy_multilabel(y_pred)
np_y = to_numpy_multilabel(y)
assert pr._type == 'multilabel'
pr_compute = pr.compute() if average else pr.compute().mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert precision_score(
np_y, np_y_pred,
average='samples') == pytest.approx(pr_compute)
# Batched Updates
pr.reset()
y_pred = torch.randint(0, 2, size=(100, 5, 12, 14))
y = torch.randint(0, 2, size=(100, 5, 12, 14)).type(torch.LongTensor)
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
pr.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
np_y = to_numpy_multilabel(y)
np_y_pred = to_numpy_multilabel(y_pred)
assert pr._type == 'multilabel'
pr_compute = pr.compute() if average else pr.compute().mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert precision_score(
np_y, np_y_pred,
average='samples') == pytest.approx(pr_compute)
for _ in range(5):
_test(average=True)
_test(average=False)
pr1 = Precision(is_multilabel=True, average=True)
pr2 = Precision(is_multilabel=True, average=False)
y_pred = torch.randint(0, 2, size=(10, 4, 20, 23))
y = torch.randint(0, 2, size=(10, 4, 20, 23)).type(torch.LongTensor)
pr1.update((y_pred, y))
pr2.update((y_pred, y))
assert pr1.compute() == pytest.approx(pr2.compute().mean().item())
