def _test(average): pr = Precision(average=average) y_pred = torch.rand(10, 5, 18, 16) y = torch.randint(0, 5, size=(10, 18, 16)).type(torch.LongTensor) pr.update((y_pred, y)) num_classes = y_pred.shape[1] np_y_pred = y_pred.argmax(dim=1).numpy().ravel() np_y = y.numpy().ravel() assert pr._type == 'multiclass' assert isinstance(pr.compute(), float if average else torch.Tensor) pr_compute = pr.compute() if average else pr.compute().numpy() sk_average_parameter = 'macro' if average else None with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) sk_compute = precision_score(np_y, np_y_pred, labels=range(0, num_classes), average=sk_average_parameter) assert sk_compute == pytest.approx(pr_compute) pr.reset() y_pred = torch.rand(10, 7, 20, 12) y = torch.randint(0, 7, size=(10, 20, 12)).type(torch.LongTensor) pr.update((y_pred, y)) num_classes = y_pred.shape[1] np_y_pred = y_pred.argmax(dim=1).numpy().ravel() np_y = y.numpy().ravel() assert pr._type == 'multiclass' assert isinstance(pr.compute(), float if average else torch.Tensor) pr_compute = pr.compute() if average else pr.compute().numpy() sk_average_parameter = 'macro' if average else None with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) sk_compute = precision_score(np_y, np_y_pred, labels=range(0, num_classes), average=sk_average_parameter) assert sk_compute == pytest.approx(pr_compute) # Batched Updates pr.reset() y_pred = torch.rand(100, 8, 12, 14) y = torch.randint(0, 8, size=(100, 12, 14)).type(torch.LongTensor) batch_size = 16 n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size pr.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) num_classes = y_pred.shape[1] np_y = y.numpy().ravel() np_y_pred = y_pred.argmax(dim=1).numpy().ravel() assert pr._type == 'multiclass' assert isinstance(pr.compute(), float if average else torch.Tensor) pr_compute = pr.compute() if average else pr.compute().numpy() sk_average_parameter = 'macro' if average else None with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) sk_compute = precision_score(np_y, np_y_pred, labels=range(0, num_classes), average=sk_average_parameter) assert sk_compute == pytest.approx(pr_compute)
def test_binary_wrong_inputs(): pr = Precision() with pytest.raises(ValueError): # y has not only 0 or 1 values pr.update((torch.randint(0, 2, size=(10, )).long(), torch.arange(0, 10).long())) with pytest.raises(ValueError): # y_pred values are not thresholded to 0, 1 values pr.update(( torch.rand(10, ), torch.randint(0, 2, size=(10, )).long(), )) with pytest.raises(ValueError): # incompatible shapes pr.update(( torch.randint(0, 2, size=(10, )).long(), torch.randint(0, 2, size=(10, 5)).long(), )) with pytest.raises(ValueError): # incompatible shapes pr.update(( torch.randint(0, 2, size=(10, 5, 6)).long(), torch.randint(0, 2, size=(10, )).long(), )) with pytest.raises(ValueError): # incompatible shapes pr.update(( torch.randint(0, 2, size=(10, )).long(), torch.randint(0, 2, size=(10, 5, 6)).long(), ))
def _test(y_pred, y, batch_size): pr.reset() if batch_size > 1: n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size pr.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size])) else: pr.update((y_pred, y)) np_y_pred = to_numpy_multilabel(y_pred) np_y = to_numpy_multilabel(y) assert pr._type == "multilabel" pr_compute = pr.compute() if average else pr.compute().mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert precision_score(np_y, np_y_pred, average="samples") == pytest.approx(pr_compute) pr1 = Precision(is_multilabel=True, average=True) pr2 = Precision(is_multilabel=True, average=False) pr1.update((y_pred, y)) pr2.update((y_pred, y)) assert pr1.compute() == pytest.approx(pr2.compute().mean().item())
def test_multilabel_wrong_inputs(): pr = Precision(average=True, is_multilabel=True) with pytest.raises(ValueError): # incompatible shapes pr.update((torch.randint(0, 2, size=(10,)), torch.randint(0, 2, size=(10,)).long())) with pytest.raises(ValueError): # incompatible y_pred pr.update((torch.rand(10, 5), torch.randint(0, 2, size=(10, 5)).long())) with pytest.raises(ValueError): # incompatible y pr.update((torch.randint(0, 5, size=(10, 5, 6)), torch.rand(10))) with pytest.raises(ValueError): # incompatible shapes between two updates pr.update((torch.randint(0, 2, size=(20, 5)), torch.randint(0, 2, size=(20, 5)).long())) pr.update((torch.randint(0, 2, size=(20, 6)), torch.randint(0, 2, size=(20, 6)).long()))
def _test(average): pr = Precision(average=average, is_multilabel=True) y_pred = torch.randint(0, 2, size=(10, 5, 18, 16)) y = torch.randint(0, 2, size=(10, 5, 18, 16)).long() pr.update((y_pred, y)) np_y_pred = to_numpy_multilabel(y_pred) np_y = to_numpy_multilabel(y) assert pr._type == "multilabel" pr_compute = pr.compute() if average else pr.compute().mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert precision_score( np_y, np_y_pred, average="samples") == pytest.approx(pr_compute) pr.reset() y_pred = torch.randint(0, 2, size=(10, 4, 20, 23)) y = torch.randint(0, 2, size=(10, 4, 20, 23)).long() pr.update((y_pred, y)) np_y_pred = to_numpy_multilabel(y_pred) np_y = to_numpy_multilabel(y) assert pr._type == "multilabel" pr_compute = pr.compute() if average else pr.compute().mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert precision_score( np_y, np_y_pred, average="samples") == pytest.approx(pr_compute) # Batched Updates pr.reset() y_pred = torch.randint(0, 2, size=(100, 5, 12, 14)) y = torch.randint(0, 2, size=(100, 5, 12, 14)).long() batch_size = 16 n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size pr.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) np_y = to_numpy_multilabel(y) np_y_pred = to_numpy_multilabel(y_pred) assert pr._type == "multilabel" pr_compute = pr.compute() if average else pr.compute().mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert precision_score( np_y, np_y_pred, average="samples") == pytest.approx(pr_compute)
def _test(average): pr = Precision(average=average, is_multilabel=True) y_pred = torch.randint(0, 2, size=(20, 5)) y = torch.randint(0, 2, size=(20, 5)).type(torch.LongTensor) pr.update((y_pred, y)) np_y_pred = y_pred.numpy() np_y = y.numpy() assert pr._type == 'multilabel' pr_compute = pr.compute() if average else pr.compute().mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert precision_score( np_y, np_y_pred, average='samples') == pytest.approx(pr_compute) pr.reset() y_pred = torch.randint(0, 2, size=(10, 4)) y = torch.randint(0, 2, size=(10, 4)).type(torch.LongTensor) pr.update((y_pred, y)) np_y_pred = y_pred.numpy() np_y = y.numpy() assert pr._type == 'multilabel' pr_compute = pr.compute() if average else pr.compute().mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert precision_score( np_y, np_y_pred, average='samples') == pytest.approx(pr_compute) # Batched Updates pr.reset() y_pred = torch.randint(0, 2, size=(100, 4)) y = torch.randint(0, 2, size=(100, 4)).type(torch.LongTensor) batch_size = 16 n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size pr.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) np_y = y.numpy() np_y_pred = y_pred.numpy() assert pr._type == 'multilabel' pr_compute = pr.compute() if average else pr.compute().mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert precision_score( np_y, np_y_pred, average='samples') == pytest.approx(pr_compute)
def test_multiclass_wrong_inputs(): pr = Precision() with pytest.raises(ValueError): # incompatible shapes pr.update((torch.rand(10, 5, 4), torch.randint(0, 2, size=(10,)).type(torch.LongTensor))) with pytest.raises(ValueError): # incompatible shapes pr.update((torch.rand(10, 5, 6), torch.randint(0, 5, size=(10, 5)).type(torch.LongTensor))) with pytest.raises(ValueError): # incompatible shapes pr.update((torch.rand(10), torch.randint(0, 5, size=(10, 5, 6)).type(torch.LongTensor)))
def _test(average): pr = Precision(average=average) y_pred = torch.rand(20, 6) y = torch.randint(0, 5, size=(20, )).type(torch.LongTensor) pr.update((y_pred, y)) np_y_pred = y_pred.numpy().argmax(axis=1).ravel() np_y = y.numpy().ravel() assert pr._type == 'multiclass' assert isinstance(pr.compute(), float if average else torch.Tensor) pr_compute = pr.compute() if average else pr.compute().numpy() sklearn_average_parameter = 'macro' if average else None with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert precision_score( np_y, np_y_pred, average=sklearn_average_parameter) == pytest.approx(pr_compute) pr.reset() y_pred = torch.rand(10, 4) y = torch.randint(0, 3, size=(10, 1)).type(torch.LongTensor) pr.update((y_pred, y)) np_y_pred = y_pred.numpy().argmax(axis=1).ravel() np_y = y.numpy().ravel() assert pr._type == 'multiclass' assert isinstance(pr.compute(), float if average else torch.Tensor) pr_compute = pr.compute() if average else pr.compute().numpy() sklearn_average_parameter = 'macro' if average else None with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert precision_score( np_y, np_y_pred, average=sklearn_average_parameter) == pytest.approx(pr_compute) # 2-classes pr.reset() y_pred = torch.rand(10, 2) y = torch.randint(0, 2, size=(10, 1)).type(torch.LongTensor) pr.update((y_pred, y)) np_y_pred = y_pred.numpy().argmax(axis=1).ravel() np_y = y.numpy().ravel() assert pr._type == 'multiclass' assert isinstance(pr.compute(), float if average else torch.Tensor) pr_compute = pr.compute() if average else pr.compute().numpy() sklearn_average_parameter = 'macro' if average else None with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert precision_score( np_y, np_y_pred, average=sklearn_average_parameter) == pytest.approx(pr_compute)
def _test(average): pr = Precision(average=average) y_pred = torch.randint(0, 2, size=(10, 12, 10)) y = torch.randint(0, 2, size=(10, 12, 10)).type(torch.LongTensor) pr.update((y_pred, y)) np_y = y.numpy().ravel() np_y_pred = y_pred.numpy().ravel() assert pr._type == 'binary' assert isinstance(pr.compute(), float if average else torch.Tensor) pr_compute = pr.compute() if average else pr.compute().numpy() assert precision_score(np_y, np_y_pred, average='binary') == pytest.approx(pr_compute) pr.reset() y_pred = torch.randint(0, 2, size=(10, 1, 12, 10)) y = torch.randint(0, 2, size=(10, 1, 12, 10)).type(torch.LongTensor) pr.update((y_pred, y)) np_y = y.numpy().ravel() np_y_pred = y_pred.numpy().ravel() assert pr._type == 'binary' assert isinstance(pr.compute(), float if average else torch.Tensor) pr_compute = pr.compute() if average else pr.compute().numpy() assert precision_score(np_y, np_y_pred, average='binary') == pytest.approx(pr_compute) pr = Precision(average=average) # Batched Updates pr.reset() y_pred = torch.randint(0, 2, size=(100, 12, 10)) y = torch.randint(0, 2, size=(100, 1, 12, 10)).type(torch.LongTensor) batch_size = 16 n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size pr.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) np_y = y.numpy().ravel() np_y_pred = y_pred.numpy().ravel() assert pr._type == 'binary' assert isinstance(pr.compute(), float if average else torch.Tensor) pr_compute = pr.compute() if average else pr.compute().numpy() assert precision_score(np_y, np_y_pred, average='binary') == pytest.approx(pr_compute)
def _test(average): pr = Precision(average=average) y_pred = torch.rand(10, 1) y = torch.randint(0, 2, size=(10, )).type(torch.LongTensor) pr.update((y_pred, y)) np_y = y.numpy().ravel() # np_y_pred = y_pred.numpy().ravel() np_y_pred = (y_pred.numpy().ravel() > 0.5).astype('int') assert pr._type == 'binary' assert isinstance(pr.compute(), float if average else torch.Tensor) pr_compute = pr.compute() if average else pr.compute().numpy() assert precision_score(np_y, np_y_pred, average='binary') == pytest.approx(pr_compute) pr.reset() # TODO: y_pred should be binary after 0.1.2 release # y_pred = torch.randint(0, 2, size=(10, )).type(torch.LongTensor) y_pred = torch.rand(10) y = torch.randint(0, 2, size=(10, )).type(torch.LongTensor) pr.update((y_pred, y)) np_y = y.numpy().ravel() # np_y_pred = y_pred.numpy().ravel() np_y_pred = (y_pred.numpy().ravel() > 0.5).astype('int') assert pr._type == 'binary' assert isinstance(pr.compute(), float if average else torch.Tensor) pr_compute = pr.compute() if average else pr.compute().numpy() assert precision_score(np_y, np_y_pred, average='binary') == pytest.approx(pr_compute) pr.reset() # TODO: y_pred should be binary after 0.1.2 release y_pred = torch.Tensor( [0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.51]) y = torch.randint(0, 2, size=(10, )).type(torch.LongTensor) pr.update((y_pred, y)) np_y = y.numpy().ravel() # np_y_pred = y_pred.numpy().ravel() np_y_pred = (y_pred.numpy().ravel() > 0.5).astype('int') assert pr._type == 'binary' assert isinstance(pr.compute(), float if average else torch.Tensor) pr_compute = pr.compute() if average else pr.compute().numpy() assert precision_score(np_y, np_y_pred, average='binary') == pytest.approx(pr_compute)
def test_compute(): precision = Precision() y_pred = torch.eye(4) y = torch.ones(4).type(torch.LongTensor) precision.update((y_pred, y)) results = list(precision.compute()) assert results[0] == 0.0 assert results[1] == 1.0 assert results[2] == 0.0 assert results[3] == 0.0 precision.reset() y_pred = torch.eye(2) y = torch.ones(2).type(torch.LongTensor) precision.update((y_pred, y)) y = torch.zeros(2).type(torch.LongTensor) precision.update((y_pred, y)) results = list(precision.compute()) assert results[0] == 0.5 assert results[1] == 0.5
def test_multiclass_wrong_inputs(): pr = Precision() with pytest.raises(ValueError): # incompatible shapes pr.update((torch.rand(10, 5, 4), torch.randint(0, 2, size=(10,)).long())) with pytest.raises(ValueError): # incompatible shapes pr.update((torch.rand(10, 5, 6), torch.randint(0, 5, size=(10, 5)).long())) with pytest.raises(ValueError): # incompatible shapes pr.update((torch.rand(10), torch.randint(0, 5, size=(10, 5, 6)).long())) pr = Precision(average=True) with pytest.raises(ValueError): # incompatible shapes between two updates pr.update((torch.rand(10, 5), torch.randint(0, 5, size=(10,)).long())) pr.update((torch.rand(10, 6), torch.randint(0, 5, size=(10,)).long())) with pytest.raises(ValueError): # incompatible shapes between two updates pr.update((torch.rand(10, 5, 12, 14), torch.randint(0, 5, size=(10, 12, 14)).long())) pr.update((torch.rand(10, 6, 12, 14), torch.randint(0, 5, size=(10, 12, 14)).long())) pr = Precision(average=False) with pytest.raises(ValueError): # incompatible shapes between two updates pr.update((torch.rand(10, 5), torch.randint(0, 5, size=(10,)).long())) pr.update((torch.rand(10, 6), torch.randint(0, 5, size=(10,)).long())) with pytest.raises(ValueError): # incompatible shapes between two updates pr.update((torch.rand(10, 5, 12, 14), torch.randint(0, 5, size=(10, 12, 14)).long())) pr.update((torch.rand(10, 6, 12, 14), torch.randint(0, 5, size=(10, 12, 14)).long()))
def _test_distrib_integration_multilabel(device): from ignite.engine import Engine rank = idist.get_rank() torch.manual_seed(12) def _test(average, n_epochs, metric_device): n_iters = 60 s = 16 n_classes = 7 offset = n_iters * s y_true = torch.randint(0, 2, size=(offset * idist.get_world_size(), n_classes, 6, 8)).to(device) y_preds = torch.randint(0, 2, size=(offset * idist.get_world_size(), n_classes, 6, 8)).to(device) def update(engine, i): return ( y_preds[i * s + rank * offset:(i + 1) * s + rank * offset, ...], y_true[i * s + rank * offset:(i + 1) * s + rank * offset, ...], ) engine = Engine(update) pr = Precision(average=average, is_multilabel=True, device=metric_device) pr.attach(engine, "pr") data = list(range(n_iters)) engine.run(data=data, max_epochs=n_epochs) assert "pr" in engine.state.metrics res = engine.state.metrics["pr"] res2 = pr.compute() if isinstance(res, torch.Tensor): res = res.cpu().numpy() res2 = res2.cpu().numpy() assert (res == res2).all() else: assert res == res2 np_y_preds = to_numpy_multilabel(y_preds) np_y_true = to_numpy_multilabel(y_true) assert pr._type == "multilabel" res = res if average else res.mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert precision_score(np_y_true, np_y_preds, average="samples") == pytest.approx(res) metric_devices = ["cpu"] if device.type != "xla": metric_devices.append(idist.device()) for _ in range(2): for metric_device in metric_devices: _test(average=True, n_epochs=1, metric_device=metric_device) _test(average=True, n_epochs=2, metric_device=metric_device) _test(average=False, n_epochs=1, metric_device=metric_device) _test(average=False, n_epochs=2, metric_device=metric_device) pr1 = Precision(is_multilabel=True, average=True) pr2 = Precision(is_multilabel=True, average=False) y_pred = torch.randint(0, 2, size=(10, 4, 20, 23)) y = torch.randint(0, 2, size=(10, 4, 20, 23)).long() pr1.update((y_pred, y)) pr2.update((y_pred, y)) assert pr1.compute() == pytest.approx(pr2.compute().mean().item())
def test(model, test_loader, experiment, hyperparams): """ Validates the model performance as LM on never-seen data. :param model: the trained model to use for prediction :param test_loader: Dataloader of testing data :param experiment: comet.ml experiment object :param hyperparams: hyperparameters dictionary :param bpe: is bpe dataset or not """ # TODO: Define loss function, total loss, and total word count loss_fn = nn.CrossEntropyLoss(ignore_index=0, reduction="sum") total_loss = 0 word_count = 0 total_correct = 0 total_words = 0 precision = Precision(average=False, is_multilabel=True) # , device = device) recall = Recall(average=False, is_multilabel=True) # , device = device) model = model.eval() with experiment.test(): # TODO: Write testing loop with torch.no_grad(): for batch in tqdm(test_loader): inputs = batch['lm_input'] labels = batch['lm_label'] word_count = sum([len(arr) for arr in inputs]) inputs = inputs.to(device) labels = labels.to(device) y_pred = model(inputs) loss = loss_fn(torch.transpose(y_pred, 1, 2), labels) total_loss += loss prbs = torch.nn.functional.softmax(y_pred, dim=2) batch_acc, batch_correct, batch_words = accuracy_func( prbs, labels, word_count) total_correct += batch_correct total_words += batch_words predicted_ids = torch.argmax(prbs, dim=2) y_pred_one_hot = F.one_hot(predicted_ids.to(torch.int64), num_classes=prbs.shape[-1]) labels_one_hot = F.one_hot(labels.to(torch.int64), num_classes=prbs.shape[-1]) precision.update((torch.transpose(y_pred_one_hot, 1, 2), torch.transpose(labels_one_hot, 1, 2))) recall.update((torch.transpose(y_pred_one_hot, 1, 2), torch.transpose(labels_one_hot, 1, 2))) perplexity = torch.exp(total_loss / total_words) accuracy = total_correct / total_words perplexity = perplexity.item() # accuracy = accuracy.cpu() F1 = (precision * recall * 2 / (precision + recall)).mean().compute().item() print("F1: ", F1) experiment.log_metric("F1", F1) print("total_correct: ", total_correct) print("total_words: ", total_words) print("perplexity:", perplexity) print("accuracy:", accuracy) experiment.log_metric("perplexity", perplexity) experiment.log_metric("accuracy", accuracy)
loss = Loss(F.cross_entropy) precision = Precision() sensitivity = Sensitivity() specificity = Specificity() for i in range(FG.fold): parser.args.cur_fold = i output, target = run_fold(parser, vis) output = torch.cat(output) target = torch.cat(target) arg = (output, target) acc.update(arg) loss.update(arg) precision.update(arg) sensitivity.update(arg) specificity.update(arg) end = '<br>' text = 'Over all result<br>' text += 'accuracy: ' + '{:.4f}'.format(acc.compute()) + end text += 'loss: ' + '{:.4f}'.format(loss.compute()) + end text += 'precision: ' + '{}'.format(precision.compute()) + end text += 'sensitivity: ' + '{}'.format(sensitivity.compute()) + end text += 'specificity: ' + '{}'.format(specificity.compute()) + end vis.text(text, 'result_overall') vis.save([vis.env])
def train_predict(dataloader_train,dataloader_val,model,epochs,learning_rate,use_cuda): start = torch.cuda.Event(enable_timing=True) end = torch.cuda.Event(enable_timing=True) if use_cuda: model = model.cuda() model = model.train() start.record() train_loss_list=[] val_loss_list=[] train_f1=[] val_f1=[] loss_fn = nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) precision = Precision() recall = Recall() f1 = Fbeta(beta=1.0, average=True, precision=precision, recall=recall) for epoch in range(epochs): print("Epoch: {}".format(epoch+1)) for i,(img, label) in enumerate(dataloader_train): img, label = Variable(img),Variable(label) if use_cuda: img = img.cuda() label = label.cuda() optimizer.zero_grad() pred = model.forward(img) _,my_label = torch.max(label, dim=1) loss = loss_fn(pred,my_label) if i == len(dataloader_train)-1: train_loss_list.append(loss.item()) loss.backward() optimizer.step() precision.update((pred, my_label)) recall.update((pred, my_label)) f1.update((pred, my_label)) print("\tTrain loss: {:0.2f}".format(train_loss_list[-1])) precision.compute() recall.compute() train_f1.append(f1.compute()*100) print("\tTrain F1 Score: {:0.2f}%".format(train_f1[-1])) precision = Precision() recall = Recall() f1 = Fbeta(beta=1.0, average=True, precision=precision, recall=recall) with torch.no_grad(): for i,(img, label) in enumerate(dataloader_val): img, labels = Variable(img),Variable(label) if use_cuda: img = img.cuda() label = label.cuda() pred = model(img) _,my_label = torch.max(label, dim=1) loss = loss_fn(pred,my_label) if i == len(dataloader_val)-1: val_loss_list.append(loss.item()) precision.update((pred, my_label)) recall.update((pred, my_label)) f1.update((pred, my_label)) print("\n\tVal loss: {:0.2f}".format(val_loss_list[-1])) precision.compute() recall.compute() val_f1.append(f1.compute()*100) print("\tVal F1 Score: {:0.2f}%".format(val_f1[-1])) end.record() torch.cuda.synchronize() time = start.elapsed_time(end) return (train_loss_list,val_loss_list,train_f1,val_f1,time,model)
def _test_distrib_integration_multilabel(device): from ignite.engine import Engine rank = idist.get_rank() torch.manual_seed(12) def _test(average, n_epochs): n_iters = 60 s = 16 n_classes = 7 offset = n_iters * s y_true = torch.randint(0, 2, size=(offset * idist.get_world_size(), n_classes, 6, 8)).to(device) y_preds = torch.randint(0, 2, size=(offset * idist.get_world_size(), n_classes, 6, 8)).to(device) def update(engine, i): return ( y_preds[i * s + rank * offset : (i + 1) * s + rank * offset, ...], y_true[i * s + rank * offset : (i + 1) * s + rank * offset, ...], ) engine = Engine(update) pr = Precision(average=average, is_multilabel=True) pr.attach(engine, "pr") data = list(range(n_iters)) engine.run(data=data, max_epochs=n_epochs) assert "pr" in engine.state.metrics res = engine.state.metrics["pr"] res2 = pr.compute() if isinstance(res, torch.Tensor): res = res.cpu().numpy() res2 = res2.cpu().numpy() assert (res == res2).all() else: assert res == res2 with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) true_res = precision_score( to_numpy_multilabel(y_true), to_numpy_multilabel(y_preds), average="samples" if average else None ) assert pytest.approx(res) == true_res for _ in range(2): _test(average=True, n_epochs=1) _test(average=True, n_epochs=2) if idist.get_world_size() > 1: with pytest.warns( RuntimeWarning, match="Precision/Recall metrics do not work in distributed setting when " "average=False and is_multilabel=True", ): pr = Precision(average=False, is_multilabel=True) y_pred = torch.randint(0, 2, size=(4, 3, 6, 8)) y = torch.randint(0, 2, size=(4, 3, 6, 8)).long() pr.update((y_pred, y)) pr_compute1 = pr.compute() pr_compute2 = pr.compute() assert len(pr_compute1) == 4 * 6 * 8 assert (pr_compute1 == pr_compute2).all()
def test_multilabel_input_NCHW(): def _test(average): pr = Precision(average=average, is_multilabel=True) y_pred = torch.randint(0, 2, size=(10, 5, 18, 16)) y = torch.randint(0, 2, size=(10, 5, 18, 16)).type(torch.LongTensor) pr.update((y_pred, y)) np_y_pred = to_numpy_multilabel(y_pred) np_y = to_numpy_multilabel(y) assert pr._type == 'multilabel' pr_compute = pr.compute() if average else pr.compute().mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert precision_score( np_y, np_y_pred, average='samples') == pytest.approx(pr_compute) pr.reset() y_pred = torch.randint(0, 2, size=(10, 4, 20, 23)) y = torch.randint(0, 2, size=(10, 4, 20, 23)).type(torch.LongTensor) pr.update((y_pred, y)) np_y_pred = to_numpy_multilabel(y_pred) np_y = to_numpy_multilabel(y) assert pr._type == 'multilabel' pr_compute = pr.compute() if average else pr.compute().mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert precision_score( np_y, np_y_pred, average='samples') == pytest.approx(pr_compute) # Batched Updates pr.reset() y_pred = torch.randint(0, 2, size=(100, 5, 12, 14)) y = torch.randint(0, 2, size=(100, 5, 12, 14)).type(torch.LongTensor) batch_size = 16 n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size pr.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) np_y = to_numpy_multilabel(y) np_y_pred = to_numpy_multilabel(y_pred) assert pr._type == 'multilabel' pr_compute = pr.compute() if average else pr.compute().mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert precision_score( np_y, np_y_pred, average='samples') == pytest.approx(pr_compute) for _ in range(5): _test(average=True) _test(average=False) pr1 = Precision(is_multilabel=True, average=True) pr2 = Precision(is_multilabel=True, average=False) y_pred = torch.randint(0, 2, size=(10, 4, 20, 23)) y = torch.randint(0, 2, size=(10, 4, 20, 23)).type(torch.LongTensor) pr1.update((y_pred, y)) pr2.update((y_pred, y)) assert pr1.compute() == pytest.approx(pr2.compute().mean().item())