def test_binary_wrong_inputs(): re = Recall() assert re._updated is False with pytest.raises( ValueError, match=r"For binary cases, y must be comprised of 0's and 1's"): # y has not only 0 or 1 values re.update((torch.randint(0, 2, size=(10, )), torch.arange(0, 10).long())) assert re._updated is False with pytest.raises( ValueError, match=r"For binary cases, y_pred must be comprised of 0's and 1's" ): # y_pred values are not thresholded to 0, 1 values re.update((torch.rand(10, 1), torch.randint(0, 2, size=(10, )).long())) assert re._updated is False with pytest.raises(ValueError, match=r"y must have shape of"): # incompatible shapes re.update((torch.randint(0, 2, size=(10, )), torch.randint(0, 2, size=(10, 5)).long())) assert re._updated is False with pytest.raises(ValueError, match=r"y must have shape of"): # incompatible shapes re.update((torch.randint(0, 2, size=(10, 5, 6)), torch.randint(0, 2, size=(10, )).long())) assert re._updated is False with pytest.raises(ValueError, match=r"y must have shape of"): # incompatible shapes re.update((torch.randint(0, 2, size=(10, )), torch.randint(0, 2, size=(10, 5, 6)).long())) assert re._updated is False
'optimizer': Adam(gru.parameters(), lr=learning_rate), 'loss': nn.CrossEntropyLoss(), 'epoch': 5, }, { 'nn': bgru, 'optimizer': Adam(bgru.parameters(), lr=learning_rate), 'loss': nn.CrossEntropyLoss(), 'epoch': 5, }, ] # set metrics metrics = { 'prec': Precision(average=True), 'recall': Recall(average=True), } # user-defined event handler def log_iter_10(trainer, evaluator, train, valid): iter_num = trainer.state.iteration epoch_num = trainer.state.epoch loss = trainer.state.output if iter_num % 10 == 0: logger.info(f"Epoch[{epoch_num}] Iter: {iter_num} Loss: {loss:.2}") def main(): # logger config logger.level = 'debug'
def test_binary_wrong_inputs(): re = Recall() with pytest.raises(ValueError): # y has not only 0 or 1 values re.update((torch.randint(0, 2, size=(10, )), torch.arange(0, 10).type(torch.LongTensor))) with pytest.raises(ValueError): # y_pred values are not thresholded to 0, 1 values re.update( (torch.rand(10, 1), torch.randint(0, 2, size=(10, )).type(torch.LongTensor))) with pytest.raises(ValueError): # incompatible shapes re.update((torch.randint(0, 2, size=(10, )), torch.randint(0, 2, size=(10, 5)).type(torch.LongTensor))) with pytest.raises(ValueError): # incompatible shapes re.update((torch.randint(0, 2, size=(10, 5, 6)), torch.randint(0, 2, size=(10, )).type(torch.LongTensor))) with pytest.raises(ValueError): # incompatible shapes re.update((torch.randint(0, 2, size=(10, )), torch.randint(0, 2, size=(10, 5, 6)).type(torch.LongTensor)))
def test_multiclass_wrong_inputs(): re = Recall() with pytest.raises(ValueError): # incompatible shapes re.update((torch.rand(10, 5, 4), torch.randint(0, 2, size=(10, )).long())) with pytest.raises(ValueError): # incompatible shapes re.update((torch.rand(10, 5, 6), torch.randint(0, 5, size=(10, 5)).long())) with pytest.raises(ValueError): # incompatible shapes re.update((torch.rand(10), torch.randint(0, 5, size=(10, 5, 6)).long())) re = Recall(average=True) with pytest.raises(ValueError): # incompatible shapes between two updates re.update((torch.rand(10, 5), torch.randint(0, 5, size=(10, )).long())) re.update((torch.rand(10, 6), torch.randint(0, 5, size=(10, )).long())) with pytest.raises(ValueError): # incompatible shapes between two updates re.update((torch.rand(10, 5, 12, 14), torch.randint(0, 5, size=(10, 12, 14)).long())) re.update((torch.rand(10, 6, 12, 14), torch.randint(0, 5, size=(10, 12, 14)).long())) re = Recall(average=False) with pytest.raises(ValueError): # incompatible shapes between two updates re.update((torch.rand(10, 5), torch.randint(0, 5, size=(10, )).long())) re.update((torch.rand(10, 6), torch.randint(0, 5, size=(10, )).long())) with pytest.raises(ValueError): # incompatible shapes between two updates re.update((torch.rand(10, 5, 12, 14), torch.randint(0, 5, size=(10, 12, 14)).long())) re.update((torch.rand(10, 6, 12, 14), torch.randint(0, 5, size=(10, 12, 14)).long()))
def _test(average): re = Recall(average=average, is_multilabel=True) y_pred = torch.randint(0, 2, size=(10, 5, 18, 16)) y = torch.randint(0, 2, size=(10, 5, 18, 16)).long() re.update((y_pred, y)) np_y_pred = to_numpy_multilabel(y_pred) np_y = to_numpy_multilabel(y) assert re._type == "multilabel" re_compute = re.compute() if average else re.compute().mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert recall_score(np_y, np_y_pred, average="samples") == pytest.approx(re_compute) re.reset() y_pred = torch.randint(0, 2, size=(10, 4, 20, 23)) y = torch.randint(0, 2, size=(10, 4, 20, 23)).long() re.update((y_pred, y)) np_y_pred = to_numpy_multilabel(y_pred) np_y = to_numpy_multilabel(y) assert re._type == "multilabel" re_compute = re.compute() if average else re.compute().mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert recall_score(np_y, np_y_pred, average="samples") == pytest.approx(re_compute) # Batched Updates re.reset() y_pred = torch.randint(0, 2, size=(100, 5, 12, 14)) y = torch.randint(0, 2, size=(100, 5, 12, 14)).long() batch_size = 16 n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size re.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) np_y = to_numpy_multilabel(y) np_y_pred = to_numpy_multilabel(y_pred) assert re._type == "multilabel" re_compute = re.compute() if average else re.compute().mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert recall_score(np_y, np_y_pred, average="samples") == pytest.approx(re_compute)
def run(tb, vb, lr, epochs, writer): device = os.environ['main-device'] logging.info('Training program start!') logging.info('Configuration:') logging.info('\n'+json.dumps(INFO, indent=2)) # ------------------------------------ # 1. Define dataloader train_loader, train4val_loader, val_loader, num_of_images, mapping = get_dataloaders(tb, vb) # train_loader, train4val_loader, val_loader, num_of_images = get_dataloaders(tb, vb) weights = (1/num_of_images)/((1/num_of_images).sum().item()) # weights = (1/num_of_images)/(1/num_of_images + 1/(num_of_images.sum().item()-num_of_images)) weights = weights.to(device=device) # ------------------------------------ # 2. Define model model = EfficientNet.from_pretrained('efficientnet-b3', num_classes=INFO['dataset-info']['num-of-classes']) model = carrier(model) # ------------------------------------ # 3. Define optimizer optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9) scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200) ignite_scheduler = LRScheduler(scheduler) # ------------------------------------ # 4. Define metrics # class SoftCrossEntropyLoss(nn.Module): # def __init__(self, weight=None): # super(SoftCrossEntropyLoss, self).__init__() # self.class_weights = weight # def forward(self, input, target): # softmax = torch.exp(input) / torch.exp(input).sum(1)[:, None] # onehot_labels = to_onehot(target, input.shape[1]) # soft_labels = torch.zeros_like(onehot_labels) # soft_labels = torch.where(onehot_labels.cpu() == 1, torch.tensor([0.9]), torch.tensor([0.1/(input.shape[1]-1)])).to(device=device) # if self.class_weights is not None: # # print(soft_labels.shape, softmax.shape) # loss = -torch.sum(torch.log(softmax) * soft_labels * self.class_weights * input.shape[1]) # else: # loss = -torch.sum(torch.log(softmax) * soft_labels) # return loss class LabelMSELoss(nn.Module): def __init__(self, weight=None): super(LabelMSELoss, self).__init__() self.class_weights = weight.to(device=device) def forward(self, input, target): target_onehot = to_onehot(target, num_classes=input.shape[1]).to(device=device) mse = (input - target_onehot) ** 2 if self.class_weights is not None: weights = self.class_weights[target] * input.shape[1] return (mse.sum(1) * weights).sum() else: return mse.sum() class EntropyPrediction(metric.Metric): def __init__(self, threshold=0.5): super(EntropyPrediction, self).__init__() self.threshold = threshold self.prediction = torch.tensor([], dtype=torch.int) self.y = torch.tensor([], dtype=torch.int) def reset(self): # self.threshold = 0.5 self.prediction = torch.tensor([]) self.y = torch.tensor([]) super(EntropyPrediction, self).reset() def update(self, output): y_pred, y = output softmax = torch.exp(y_pred) / torch.exp(y_pred).sum(1)[:, None] entropy_base = math.log(y_pred.shape[1]) entropy = (-softmax * torch.log(softmax)).sum(1)/entropy_base values, inds = softmax.max(1) prediction = torch.where(entropy<self.threshold, inds, torch.tensor([-1]).to(device=device)) self.prediction = torch.cat((self.prediction.type(torch.LongTensor).to(device=device), torch.tensor([mapping[x.item()] for x in prediction]).to(device=device))) self.y = torch.cat((self.y.type(torch.LongTensor).to(device=device), y.to(device=device))) # return self.prediction, self.y def compute(self): return self.prediction, self.y train_metrics = { 'accuracy': Accuracy(), 'loss': Loss(LabelMSELoss()), 'precision_recall': MetricsLambda(PrecisionRecallTable, Precision(), Recall(), train_loader.dataset.classes), 'cmatrix': MetricsLambda(CMatrixTable, ConfusionMatrix(INFO['dataset-info']['num-of-classes']), train_loader.dataset.classes) } val_metrics = { 'accuracy': MetricsLambda(Labels2Acc, EntropyPrediction(1.0)), 'precision_recall': MetricsLambda(Labels2PrecisionRecall, EntropyPrediction(1.0), val_loader.dataset.classes), 'cmatrix': MetricsLambda(Labels2CMatrix, EntropyPrediction(1.0), val_loader.dataset.classes) } # ------------------------------------ # 5. Create trainer trainer = create_supervised_trainer(model, optimizer, LabelMSELoss(), device=device) # ------------------------------------ # 6. Create evaluator train_evaluator = create_supervised_evaluator(model, metrics=train_metrics, device=device) val_evaluator = create_supervised_evaluator(model, metrics=val_metrics, device=device) desc = 'ITERATION - loss: {:.4f}' pbar = tqdm( initial=0, leave=False, total=len(train_loader), desc=desc.format(0) ) # ------------------------------------ # 7. Create event hooks # Update process bar on each iteration completed. @trainer.on(Events.ITERATION_COMPLETED) def log_training_loss(engine): log_interval = 1 iter = (engine.state.iteration - 1) % len(train_loader) + 1 if iter % log_interval == 0: pbar.desc = desc.format(engine.state.output) pbar.update(log_interval) @trainer.on(Events.EPOCH_STARTED) def refresh_pbar(engine): pbar.refresh() pbar.n = pbar.last_print_n = 0 # Compute metrics on train data on each epoch completed. @trainer.on(Events.EPOCH_COMPLETED) def log_training_results(engine): pbar.refresh() print ('Checking on training set.') train_evaluator.run(train4val_loader) metrics = train_evaluator.state.metrics avg_accuracy = metrics['accuracy'] avg_loss = metrics['loss'] precision_recall = metrics['precision_recall'] cmatrix = metrics['cmatrix'] prompt = """ Training Results - Epoch: {} Avg accuracy: {:.4f} Avg loss: {:.4f} precision_recall: \n{} confusion matrix: \n{} """.format(engine.state.epoch,avg_accuracy,avg_loss,precision_recall['pretty'],cmatrix['pretty']) tqdm.write(prompt) logging.info('\n'+prompt) writer.add_text(os.environ['run-id'], prompt, engine.state.epoch) writer.add_scalars('Aggregate/Acc', {'Train Acc': avg_accuracy}, engine.state.epoch) writer.add_scalars('Aggregate/Loss', {'Train Loss': avg_loss}, engine.state.epoch) # Compute metrics on val data on each epoch completed. @trainer.on(Events.EPOCH_COMPLETED) def log_validation_results(engine): pbar.clear() print('* - * - * - * - * - * - * - * - * - * - * - * - *') print ('Checking on validation set.') val_evaluator.run(val_loader) metrics = val_evaluator.state.metrics avg_accuracy = metrics['accuracy'] precision_recall = metrics['precision_recall'] cmatrix = metrics['cmatrix'] prompt = """ Validating Results - Epoch: {} Avg accuracy: {:.4f} precision_recall: \n{} confusion matrix: \n{} """.format(engine.state.epoch,avg_accuracy,precision_recall['pretty'],cmatrix['pretty']) tqdm.write(prompt) logging.info('\n'+prompt) writer.add_text(os.environ['run-id'], prompt, engine.state.epoch) writer.add_scalars('Aggregate/Acc', {'Val Acc': avg_accuracy}, engine.state.epoch) writer.add_scalars('Aggregate/Score', {'Val avg precision': precision_recall['data'][0, -1], 'Val avg recall': precision_recall['data'][1, -1]}, engine.state.epoch) # Save model ever N epoch. save_model_handler = ModelCheckpoint(os.environ['savedir'], '', save_interval=10, n_saved=2) trainer.add_event_handler(Events.EPOCH_COMPLETED, save_model_handler, {'model': model}) # Update learning-rate due to scheduler. trainer.add_event_handler(Events.EPOCH_STARTED, ignite_scheduler) # ------------------------------------ # Run trainer.run(train_loader, max_epochs=epochs) pbar.close()
def test_no_update(): recall = Recall() with pytest.raises(NotComputableError): recall.compute()
batch=batch, train_step=train_step, device="cpu") print("SUCCESS: variables changed") except Exception as e: print("FAILED: ", e) exit(1) test_variables_change(model, model_data[0]) trainer = Engine(update_model) evaluator = Engine(predict_on_batch) Accuracy().attach(evaluator, "accuracy") Precision().attach(evaluator, "precision") Recall().attach(evaluator, "recall") @trainer.on(Events.ITERATION_COMPLETED(every=100)) def log_training(engine): batch_loss = engine.state.output lr = optimizer.param_groups[0]['lr'] epoch = engine.state.epoch max_epochs = engine.state.max_epochs iteration = engine.state.iteration print( f"Epoch {epoch}/{max_epochs} : {iteration} - batch loss: {batch_loss: .4f}, lr: {lr}" ) @trainer.on(Events.EPOCH_COMPLETED)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') home = os.environ['HOME'] writer = SummaryWriter(home + '/log.json') lowest_loss = np.Inf train_loader, val_loader, test_loader = load_data(args.batch_size) model.to(device) if args.summary: summary(model, (1, 28, 28)) print('Batch size: ', args.batch_size) print('Epochs:', args.num_epochs) trainer = create_supervised_trainer(model, adam_optimizer, cross_entropy_loss, device=device) evaluator = create_supervised_evaluator(model, metrics={"accuracy": Accuracy(), "cross": Loss(cross_entropy_loss), "prec": Precision(), "recall": Recall()}, device=device) desc = "ITERATION - loss: {:.2f}" pbar = tqdm( initial=0, leave=False, total=len(train_loader), desc=desc.format(0) ) @trainer.on(Events.ITERATION_COMPLETED) def log_training_loss(engine): iter = (engine.state.iteration - 1) % len(train_loader) + 1 if iter % 10 == 0: pbar.desc = desc.format(engine.state.output)
def create_supervised_evaluator( model: torch.nn.Module, prepare_batch, criterion, metrics=None, device=None, non_blocking: bool = False, tqdm_log: bool = False, checkpoint_dir='output/checkpoints/' ) -> Engine: if device: model.to(device) def _inference(engine, batch): model.eval() with torch.no_grad(): actions, target = prepare_batch( batch, device=device, non_blocking=non_blocking) scores = model(actions) return (scores, target) engine = Engine(_inference) softmax_transform = lambda x:\ (F.softmax(x[0], dim=1)[:, 1] > 0.5, x[1]) Loss( criterion, output_transform=lambda x: x, ).attach(engine, 'loss') ROC_AUC( output_transform=lambda x: (F.softmax(x[0], dim=1)[:, 1], x[1]) ).attach(engine, 'roc_auc') ModdedPrecision( output_transform=softmax_transform ).attach(engine, 'precision') Recall( output_transform=softmax_transform ).attach(engine, 'recall') FalsePositiveRate( output_transform=softmax_transform ).attach(engine, 'FPR') if tqdm_log: pbar = ProgressBar(persist=True) pbar.attach(engine) # save the best model # to_save = {'model': model} # best_checkpoint_handler = Checkpoint( # to_save, # DiskSaver(checkpoint_dir, create_dir=True), # n_saved=1, # filename_prefix='best', # score_function=lambda x: engine.state.metrics['roc_auc'], # score_name="roc_auc", # global_step_transform=lambda x, y : engine.train_epoch) # engine.add_event_handler(Events.COMPLETED, best_checkpoint_handler) @engine.on(Events.COMPLETED) def log_validation_results(engine): metrics = engine.state.metrics if len(metrics) == 0: print('no metrics in log_validation_results!') return print(f"{'Validation Results':20} - " f"Avg loss: {metrics['loss']:.6f}, " f"ROC AUC: {metrics['roc_auc']:.6f}\n\t" f"Recall: {metrics['recall']:.6f} " f"Precision: {metrics['precision']:.6f} " f"FPR: {metrics['FPR']:.6f} " ) wandb.log({ "val_loss": metrics['loss'], "val_roc_auc": metrics['roc_auc'], "val_recall": metrics['recall'], "val_precision": metrics['precision'], "val_fpr": metrics['FPR'] }, commit=True) return engine
def create_supervised_trainer(model, optimizer, criterion, prepare_batch, metrics={}, device=None, tqdm_log=False, ) -> Engine: def _update(engine, batch): model.train() optimizer.zero_grad() actions, target = prepare_batch(batch, device=device) scores = model(actions) loss = criterion(scores, target) loss.backward() optimizer.step() return {'loss': loss.item(), 'y_pred': scores, 'y_true': target} model.to(device) engine = Engine(_update) softmax_transform = lambda x:\ (F.softmax(x['y_pred'], dim=1)[:, 1] > 0.5, x['y_true']) # Metrics RunningAverage(output_transform=lambda x: x['loss'])\ .attach(engine, 'running_average_loss') Loss( criterion, output_transform=lambda x: (x['y_pred'], x['y_true']), ).attach(engine, 'loss') ROC_AUC( output_transform=lambda x: (F.softmax(x['y_pred'], dim=1)[:, 1], x['y_true']) ).attach(engine, 'roc_auc') ModdedPrecision( output_transform=softmax_transform ).attach(engine, 'precision') Recall( output_transform=softmax_transform ).attach(engine, 'recall') FalsePositiveRate( output_transform=softmax_transform ).attach(engine, 'FPR') # TQDM if tqdm_log: pbar = ProgressBar( persist=True, ) pbar.attach(engine, ['average_loss']) @engine.on(Events.EPOCH_COMPLETED) def log_validation_results(engine): metrics = engine.state.metrics print(f"Epoch {engine.state.epoch} completed!") print(f"{'Train Results':20} - " f"Avg loss: {metrics['loss']:.6f}, " f"ROC AUC: {metrics['roc_auc']:.6f}\n\t" f"Recall: {metrics['recall']:.6f} " f"Precision: {metrics['precision']:.6f} " f"FPR: {metrics['FPR']:.6f} " ) wandb.log({ "train_loss": metrics['loss'], "train_roc_auc": metrics['roc_auc'], "train_recall": metrics['recall'], "train_precision": metrics['precision'], "train_fpr": metrics['FPR'] }, commit=False) return engine
def train(name, load, lrate, weight_decay, workers, smooth, device, validation, ground_truth): if not name: name = '{}_{}'.format(lrate, weight_decay) click.echo('model output name: {}'.format(name)) torch.set_num_threads(1) train_set = BaselineSet(glob.glob('{}/**/*.seeds.png'.format(ground_truth), recursive=True), smooth=smooth) train_data_loader = DataLoader(dataset=train_set, num_workers=workers, batch_size=1, shuffle=True, pin_memory=True) val_set = BaselineSet(glob.glob('{}/**/*.seeds.png'.format(validation), recursive=True), smooth=smooth) val_data_loader = DataLoader(dataset=val_set, num_workers=workers, batch_size=1, pin_memory=True) click.echo('loading network') model = ResUNet(refine_encoder=False).to(device) if load: click.echo('loading weights') model = torch.load(load, map_location=device) criterion = nn.BCEWithLogitsLoss() opti = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=lrate, weight_decay=weight_decay) def score_function(engine): val_loss = engine.state.metrics['loss'] return -val_loss def output_preprocess(output): o, target = output o = torch.sigmoid(o) o = denoising_hysteresis_thresh(o.detach().squeeze().cpu().numpy(), 0.8, 0.9, 2.5) return torch.from_numpy(o.astype('f')).unsqueeze(0).unsqueeze(0).to( device), target.double().to(device) trainer = create_supervised_trainer(model, opti, criterion, device=device, non_blocking=True) accuracy = Accuracy(output_transform=output_preprocess) precision = Precision(output_transform=output_preprocess) recall = Recall(output_transform=output_preprocess) loss = Loss(criterion) precision = Precision(average=False) recall = Recall(average=False) f1 = (precision * recall * 2 / (precision + recall)).mean() evaluator = create_supervised_evaluator(model, device=device, non_blocking=True) accuracy.attach(evaluator, 'accuracy') precision.attach(evaluator, 'precision') recall.attach(evaluator, 'recall') loss.attach(evaluator, 'loss') f1.attach(evaluator, 'f1') ckpt_handler = ModelCheckpoint('.', name, save_interval=1, n_saved=10, require_empty=False) RunningAverage(output_transform=lambda x: x).attach(trainer, 'loss') progress_bar = ProgressBar(persist=True) progress_bar.attach(trainer, ['loss']) trainer.add_event_handler(event_name=Events.EPOCH_COMPLETED, handler=ckpt_handler, to_save={'net': model}) trainer.add_event_handler(event_name=Events.ITERATION_COMPLETED, handler=TerminateOnNan()) @trainer.on(Events.EPOCH_COMPLETED) def log_validation_results(engine): evaluator.run(val_data_loader) metrics = evaluator.state.metrics progress_bar.log_message( 'eval results - epoch {} loss: {:.4f} f1: {:.4f}, accuracy: {:.4f} recall: {:.4f} precision {:.4f}' .format(engine.state.epoch, metrics['loss'], metrics['f1'], metrics['accuracy'], metrics['recall'], metrics['precision'])) trainer.run(train_data_loader, max_epochs=1000)
def test_binary_input(average): re = Recall(average=average) assert re._updated is False def _test(y_pred, y, batch_size): re.reset() assert re._updated is False if batch_size > 1: n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size re.update( (y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) else: re.update((y_pred, y)) np_y = y.numpy().ravel() np_y_pred = y_pred.numpy().ravel() assert re._type == "binary" assert re._updated is True assert isinstance(re.compute(), float if average else torch.Tensor) re_compute = re.compute() if average else re.compute().numpy() assert recall_score(np_y, np_y_pred, average="binary") == pytest.approx(re_compute) def get_test_cases(): test_cases = [ # Binary accuracy on input of shape (N, 1) or (N, ) (torch.randint(0, 2, size=(10, )), torch.randint(0, 2, size=(10, )), 1), (torch.randint(0, 2, size=(10, 1)), torch.randint(0, 2, size=(10, 1)), 1), # updated batches (torch.randint(0, 2, size=(50, )), torch.randint(0, 2, size=(50, )), 16), (torch.randint(0, 2, size=(50, 1)), torch.randint(0, 2, size=(50, 1)), 16), # Binary accuracy on input of shape (N, L) (torch.randint(0, 2, size=(10, 5)), torch.randint(0, 2, size=(10, 5)), 1), (torch.randint(0, 2, size=(10, 1, 5)), torch.randint(0, 2, size=(10, 1, 5)), 1), # updated batches (torch.randint(0, 2, size=(50, 5)), torch.randint(0, 2, size=(50, 5)), 16), (torch.randint(0, 2, size=(50, 1, 5)), torch.randint(0, 2, size=(50, 1, 5)), 16), # Binary accuracy on input of shape (N, H, W) (torch.randint(0, 2, size=(10, 12, 10)), torch.randint(0, 2, size=(10, 12, 10)), 1), (torch.randint(0, 2, size=(10, 1, 12, 10)), torch.randint(0, 2, size=(10, 1, 12, 10)), 1), # updated batches (torch.randint(0, 2, size=(50, 12, 10)), torch.randint(0, 2, size=(50, 12, 10)), 16), (torch.randint(0, 2, size=(50, 1, 12, 10)), torch.randint(0, 2, size=(50, 1, 12, 10)), 16), # Corner case with all zeros predictions (torch.zeros(size=(10, )), torch.randint(0, 2, size=(10, )), 1), (torch.zeros(size=(10, 1)), torch.randint(0, 2, size=(10, 1)), 1), ] return test_cases for _ in range(5): # check multiple random inputs as random exact occurencies are rare test_cases = get_test_cases() for y_pred, y, batch_size in test_cases: _test(y_pred, y, batch_size)
def _test_distrib_integration_multilabel(device): from ignite.engine import Engine rank = idist.get_rank() torch.manual_seed(12) def _test(average, n_epochs, metric_device): n_iters = 60 s = 16 n_classes = 7 offset = n_iters * s y_true = torch.randint(0, 2, size=(offset * idist.get_world_size(), n_classes, 6, 8)).to(device) y_preds = torch.randint(0, 2, size=(offset * idist.get_world_size(), n_classes, 6, 8)).to(device) def update(engine, i): return ( y_preds[i * s + rank * offset:(i + 1) * s + rank * offset, ...], y_true[i * s + rank * offset:(i + 1) * s + rank * offset, ...], ) engine = Engine(update) re = Recall(average=average, is_multilabel=True, device=metric_device) re.attach(engine, "re") assert re._updated is False data = list(range(n_iters)) engine.run(data=data, max_epochs=n_epochs) assert "re" in engine.state.metrics assert re._updated is True res = engine.state.metrics["re"] res2 = re.compute() if isinstance(res, torch.Tensor): res = res.cpu().numpy() res2 = res2.cpu().numpy() assert (res == res2).all() else: assert res == res2 np_y_preds = to_numpy_multilabel(y_preds) np_y_true = to_numpy_multilabel(y_true) assert re._type == "multilabel" res = res if average else res.mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert recall_score(np_y_true, np_y_preds, average="samples") == pytest.approx(res) metric_devices = ["cpu"] if device.type != "xla": metric_devices.append(idist.device()) for _ in range(2): for metric_device in metric_devices: _test(average=True, n_epochs=1, metric_device=metric_device) _test(average=True, n_epochs=2, metric_device=metric_device) _test(average=False, n_epochs=1, metric_device=metric_device) _test(average=False, n_epochs=2, metric_device=metric_device) re1 = Recall(is_multilabel=True, average=True) re2 = Recall(is_multilabel=True, average=False) assert re1._updated is False assert re2._updated is False y_pred = torch.randint(0, 2, size=(10, 4, 20, 23)) y = torch.randint(0, 2, size=(10, 4, 20, 23)).long() re1.update((y_pred, y)) re2.update((y_pred, y)) assert re1._updated is True assert re2._updated is True assert re1.compute() == pytest.approx(re2.compute().mean().item())
def Fbeta(beta, average=True, precision=None, recall=None, output_transform=None, device=None): """Calculates F-beta score Args: beta (float): weight of precision in harmonic mean average (bool, optional): if True, F-beta score is computed as the unweighted average (across all classes in multiclass case), otherwise, returns a tensor with F-beta score for each class in multiclass case. precision (Precision, optional): precision object metric with `average=False` to compute F-beta score recall (Precision, optional): recall object metric with `average=False` to compute F-beta score output_transform (callable, optional): a callable that is used to transform the :class:`~ignite.engine.Engine`'s `process_function`'s output into the form expected by the metric. It is used only if precision or recall are not provided. device (str of torch.device, optional): device specification in case of distributed computation usage. In most of the cases, it can be defined as "cuda:local_rank" or "cuda" if already set `torch.cuda.set_device(local_rank)`. By default, if a distributed process group is initialized and available, device is set to `cuda`. Returns: MetricsLambda, F-beta metric """ if not (beta > 0): raise ValueError( "Beta should be a positive integer, but given {}".format(beta)) if precision is not None and output_transform is not None: raise ValueError( "If precision argument is provided, output_transform should be None" ) if recall is not None and output_transform is not None: raise ValueError( "If recall argument is provided, output_transform should be None") if precision is None: precision = Precision(output_transform=(lambda x: x) if output_transform is None else output_transform, average=False, device=device) elif precision._average: raise ValueError("Input precision metric should have average=False") if recall is None: recall = Recall(output_transform=(lambda x: x) if output_transform is None else output_transform, average=False, device=device) elif recall._average: raise ValueError("Input recall metric should have average=False") fbeta = (1.0 + beta**2) * precision * recall / (beta**2 * precision + recall + 1e-15) if average: fbeta = fbeta.mean().item() return fbeta
def run_training(model, train, valid, optimizer, loss, lr_find=False): print_file(f'Experiment: {rcp.experiment}\nDescription:{rcp.description}', f'{rcp.base_path}description.txt') print_file(model, f'{rcp.models_path}model.txt') print_file(get_transforms(), f'{rcp.models_path}transform_{rcp.stage}.txt') # Data train.transform = get_transforms() valid.transform = get_transforms() train.save_csv(f'{rcp.base_path}train_df_{rcp.stage}.csv') valid.save_csv(f'{rcp.base_path}valid_df_{rcp.stage}.csv') train_loader = DataLoader(train, batch_size=rcp.bs, num_workers=8, shuffle=rcp.shuffle_batch) valid_loader = DataLoader(valid, batch_size=rcp.bs, num_workers=8, shuffle=rcp.shuffle_batch) if lr_find: lr_finder(model, optimizer, loss, train_loader, valid_loader) one_batch = next(iter(train_loader)) dot = make_dot(model(one_batch[0].to(cfg.device)), params=dict(model.named_parameters())) dot.render(f'{rcp.models_path}graph', './', format='png', cleanup=True) summary(model, one_batch[0].shape[-3:], batch_size=rcp.bs, device=cfg.device, to_file=f'{rcp.models_path}summary_{rcp.stage}.txt') # Engines trainer = create_supervised_trainer(model, optimizer, loss, device=cfg.device) t_evaluator = create_supervised_evaluator(model, metrics={ 'accuracy': Accuracy(), 'nll': Loss(loss), 'precision': Precision(average=True), 'recall': Recall(average=True), 'topK': TopKCategoricalAccuracy() }, device=cfg.device) v_evaluator = create_supervised_evaluator( model, metrics={ 'accuracy': Accuracy(), 'nll': Loss(loss), 'precision_avg': Precision(average=True), 'recall_avg': Recall(average=True), 'topK': TopKCategoricalAccuracy(), 'conf_mat': ConfusionMatrix(num_classes=len(valid.classes), average=None), }, device=cfg.device) # Tensorboard tb_logger = TensorboardLogger(log_dir=f'{rcp.tb_log_path}{rcp.stage}') tb_writer = tb_logger.writer tb_logger.attach(trainer, log_handler=OptimizerParamsHandler(optimizer, "lr"), event_name=Events.EPOCH_STARTED) tb_logger.attach(trainer, log_handler=WeightsHistHandler(model), event_name=Events.EPOCH_COMPLETED) tb_logger.attach(trainer, log_handler=WeightsScalarHandler(model), event_name=Events.ITERATION_COMPLETED) tb_logger.attach(trainer, log_handler=GradsScalarHandler(model), event_name=Events.ITERATION_COMPLETED) tb_logger.attach(trainer, log_handler=GradsHistHandler(model), event_name=Events.EPOCH_COMPLETED) @trainer.on(Events.EPOCH_COMPLETED) def tb_and_log_training_stats(engine): t_evaluator.run(train_loader) v_evaluator.run(valid_loader) tb_and_log_train_valid_stats(engine, t_evaluator, v_evaluator, tb_writer) @trainer.on( Events.ITERATION_COMPLETED(every=int(1 + len(train_loader) / 100))) def print_dash(engine): print('-', sep='', end='', flush=True) if cfg.show_batch_images: @trainer.on(Events.STARTED) def show_batch_images(engine): imgs, lbls = next(iter(train_loader)) denormalize = DeNormalize(**rcp.transforms.normalize) for i in range(len(imgs)): imgs[i] = denormalize(imgs[i]) imgs = imgs.to(cfg.device) grid = thv.utils.make_grid(imgs) tb_writer.add_image('images', grid, 0) tb_writer.add_graph(model, imgs) tb_writer.flush() if cfg.show_top_losses: @trainer.on(Events.COMPLETED) def show_top_losses(engine, k=6): nll_loss = nn.NLLLoss(reduction='none') df = predict_dataset(model, valid, nll_loss, transform=None, bs=rcp.bs, device=cfg.device) df.sort_values('loss', ascending=False, inplace=True) df.reset_index(drop=True, inplace=True) for i, row in df.iterrows(): img = cv2.imread(str(row['fname'])) img = th.as_tensor(img.transpose(2, 0, 1)) # #CHW tag = f'TopLoss_{engine.state.epoch}/{row.loss:.4f}/{row.target}/{row.pred}/{row.pred2}' tb_writer.add_image(tag, img, 0) if i >= k - 1: break tb_writer.flush() if cfg.tb_projector: images, labels = train.select_n_random(250) # get the class labels for each image class_labels = [train.classes[lab] for lab in labels] # log embeddings features = images.view(-1, images.shape[-1] * images.shape[-2]) tb_writer.add_embedding(features, metadata=class_labels, label_img=images) if cfg.log_pr_curve: @trainer.on(Events.COMPLETED) def log_pr_curve(engine): """ 1. gets the probability predictions in a test_size x num_classes Tensor 2. gets the preds in a test_size Tensor takes ~10 seconds to run """ class_probs = [] class_preds = [] with th.no_grad(): for data in valid_loader: imgs, lbls = data imgs, lbls = imgs.to(cfg.device), lbls.to(cfg.device) output = model(imgs) class_probs_batch = [ th.softmax(el, dim=0) for el in output ] _, class_preds_batch = th.max(output, 1) class_probs.append(class_probs_batch) class_preds.append(class_preds_batch) test_probs = th.cat([th.stack(batch) for batch in class_probs]) test_preds = th.cat(class_preds) for i in range(len(valid.classes)): """ Takes in a "class_index" from 0 to 9 and plots the corresponding precision-recall curve""" tensorboard_preds = test_preds == i tensorboard_probs = test_probs[:, i] tb_writer.add_pr_curve(f'{rcp.stage}/{valid.classes[i]}', tensorboard_preds, tensorboard_probs, global_step=engine.state.epoch, num_thresholds=127) tb_writer.flush() print() if cfg.lr_scheduler: # lr_scheduler = ReduceLROnPlateau(optimizer, 'min', patience=5, factor=.5, min_lr=1e-7, verbose=True) # v_evaluator.add_event_handler(Events.EPOCH_COMPLETED, lambda engine: lr_scheduler.step(v_evaluator.state.metrics['nll'])) lr_scheduler = DelayedCosineAnnealingLR(optimizer, 10, 5) trainer.add_event_handler( Events.EPOCH_COMPLETED, lambda engine: lr_scheduler.step(trainer.state.epoch)) if cfg.early_stopping: def score_function(engine): score = -1 * round(engine.state.metrics['nll'], 5) # score = engine.state.metrics['accuracy'] return score es_handler = EarlyStopping(patience=10, score_function=score_function, trainer=trainer) v_evaluator.add_event_handler(Events.COMPLETED, es_handler) if cfg.save_last_checkpoint: @trainer.on(Events.EPOCH_COMPLETED(every=1)) def save_last_checkpoint(engine): checkpoint = {} objects = {'model': model, 'optimizer': optimizer} if cfg.lr_scheduler: objects['lr_scheduler'] = lr_scheduler for k, obj in objects.items(): checkpoint[k] = obj.state_dict() th.save(checkpoint, f'{rcp.models_path}last_{rcp.stage}_checkpoint.pth') if cfg.save_best_checkpoint: def score_function(engine): score = -1 * round(engine.state.metrics['nll'], 5) # score = engine.state.metrics['accuracy'] return score objects = {'model': model, 'optimizer': optimizer} if cfg.lr_scheduler: objects['lr_scheduler'] = lr_scheduler save_best = Checkpoint( objects, DiskSaver(f'{rcp.models_path}', require_empty=False, create_dir=True), n_saved=4, filename_prefix=f'best_{rcp.stage}', score_function=score_function, score_name='val_loss', global_step_transform=global_step_from_engine(trainer)) v_evaluator.add_event_handler(Events.EPOCH_COMPLETED(every=1), save_best) load_checkpoint = False if load_checkpoint: resume_epoch = 6 cp = f'{rcp.models_path}last_{rcp.stage}_checkpoint.pth' obj = th.load(f'{cp}') Checkpoint.load_objects(objects, obj) @trainer.on(Events.STARTED) def resume_training(engine): engine.state.iteration = (resume_epoch - 1) * len( engine.state.dataloader) engine.state.epoch = resume_epoch - 1 if cfg.save_confusion_matrix: @trainer.on(Events.STARTED) def init_best_loss(engine): engine.state.metrics['best_loss'] = 1e99 @trainer.on(Events.EPOCH_COMPLETED) def confusion_matric(engine): if engine.state.metrics['best_loss'] > v_evaluator.state.metrics[ 'nll']: engine.state.metrics['best_loss'] = v_evaluator.state.metrics[ 'nll'] cm = v_evaluator.state.metrics['conf_mat'] cm_df = pd.DataFrame(cm.numpy(), index=valid.classes, columns=valid.classes) pretty_plot_confusion_matrix( cm_df, f'{rcp.results_path}cm_{rcp.stage}_{trainer.state.epoch}.png', False) if cfg.log_stats: class Hook: def __init__(self, module): self.name = module[0] self.hook = module[1].register_forward_hook(self.hook_fn) self.stats_mean = 0 self.stats_std = 0 def hook_fn(self, module, input, output): self.stats_mean = output.mean() self.stats_std = output.std() def close(self): self.hook.remove() hookF = [Hook(layer) for layer in list(model.cnn.named_children())] @trainer.on(Events.ITERATION_COMPLETED) def log_stats(engine): std = {} mean = {} for hook in hookF: tb_writer.add_scalar(f'std/{hook.name}', hook.stats_std, engine.state.iteration) tb_writer.add_scalar(f'mean/{hook.name}', hook.stats_mean, engine.state.iteration) cfg.save_yaml() rcp.save_yaml() print(f'# batches: train: {len(train_loader)}, valid: {len(valid_loader)}') trainer.run(data=train_loader, max_epochs=rcp.max_epochs) tb_writer.close() tb_logger.close() return model
def run(tb, vb, lr, epochs, writer): device = os.environ['main-device'] logging.info('Training program start!') logging.info('Configuration:') logging.info('\n'+json.dumps(INFO, indent=2)) # ------------------------------------ # 1. Define dataloader train_loader, train4val_loader, val_loader, num_of_images, mapping = get_dataloaders(tb, vb) weights = (1/num_of_images)/((1/num_of_images).sum().item()) # weights = (1/num_of_images)/(1/num_of_images + 1/(num_of_images.sum().item()-num_of_images)) weights = weights.to(device=device) # ------------------------------------ # 2. Define model model = EfficientNet.from_pretrained('efficientnet-b5', num_classes=INFO['dataset-info']['num-of-classes']) model = carrier(model) # ------------------------------------ # 3. Define optimizer optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9) scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs) ignite_scheduler = LRScheduler(scheduler) # ------------------------------------ # 4. Define metrics train_metrics = { 'accuracy': Accuracy(), 'loss': Loss(CrossEntropywithLS(weight=weights)), 'precision_recall': MetricsLambda(PrecisionRecallTable, Precision(), Recall(), train_loader.dataset.classes), 'cmatrix': MetricsLambda(CMatrixTable, ConfusionMatrix(INFO['dataset-info']['num-of-classes']), train_loader.dataset.classes) } # ------------------------------------ # 5. Create trainer trainer = create_supervised_trainer(model, optimizer, CrossEntropywithLS(weight=weights), device=device) # ------------------------------------ # 6. Create evaluator train_evaluator = create_supervised_evaluator(model, metrics=train_metrics, device=device) desc = 'ITERATION - loss: {:.4f}' pbar = tqdm( initial=0, leave=False, total=len(train_loader), desc=desc.format(0) ) # ------------------------------------ # 7. Create event hooks # Update process bar on each iteration completed. @trainer.on(Events.ITERATION_COMPLETED) def log_training_loss(engine): log_interval = 1 iter = (engine.state.iteration - 1) % len(train_loader) + 1 if iter % log_interval == 0: pbar.desc = desc.format(engine.state.output) pbar.update(log_interval) # Refresh Process bar. @trainer.on(Events.EPOCH_COMPLETED) def refresh_pbar(engine): print ('Epoch {} completed!'.format(engine.state.epoch)) pbar.refresh() pbar.n = pbar.last_print_n = 0 # Compute metrics on train data on each epoch completed. # cpe = CustomPeriodicEvent(n_epochs=50) # cpe.attach(trainer) # @trainer.on(cpe.Events.EPOCHS_50_COMPLETED) def log_training_results(engine): pbar.refresh() print ('Checking on training set.') train_evaluator.run(train4val_loader) metrics = train_evaluator.state.metrics avg_accuracy = metrics['accuracy'] avg_loss = metrics['loss'] precision_recall = metrics['precision_recall'] cmatrix = metrics['cmatrix'] prompt = """ Id: {} Training Results - Epoch: {} Avg accuracy: {:.4f} Avg loss: {:.4f} precision_recall: \n{} confusion matrix: \n{} """.format(os.environ['run-id'],engine.state.epoch,avg_accuracy,avg_loss,precision_recall['pretty'],cmatrix['pretty']) tqdm.write(prompt) logging.info('\n'+prompt) writer.add_text(os.environ['run-id'], prompt, engine.state.epoch) writer.add_scalars('Aggregate/Acc', {'Train Acc': avg_accuracy}, engine.state.epoch) writer.add_scalars('Aggregate/Loss', {'Train Loss': avg_loss}, engine.state.epoch) # pbar.n = pbar.last_print_n = 0 cpe = CustomPeriodicEvent(n_epochs=50) cpe.attach(trainer) # @trainer.on(cpe.Events.EPOCHS_50_COMPLETED) trainer.add_event_handler(cpe.Events.EPOCHS_50_COMPLETED, log_training_results) trainer.add_event_handler(Events.STARTED, log_training_results) # Save model ever N epoch. save_model_handler = ModelCheckpoint(os.environ['savedir'], '', save_interval=10, n_saved=2) trainer.add_event_handler(Events.EPOCH_COMPLETED, save_model_handler, {'model': model}) # Update learning-rate due to scheduler. trainer.add_event_handler(Events.EPOCH_STARTED, ignite_scheduler) # ------------------------------------ # Run trainer.run(train_loader, max_epochs=epochs) pbar.close()
def test_pytorch_operators(): def _test(composed_metric, metric_name, compute_true_value_fn): metrics = { metric_name: composed_metric, } y_pred = torch.rand(15, 10, 5).float() y = torch.randint(0, 5, size=(15, 10)).long() def update_fn(engine, batch): y_pred, y = batch return y_pred, y validator = Engine(update_fn) for name, metric in metrics.items(): metric.attach(validator, name) def data(y_pred, y): for i in range(y_pred.shape[0]): yield (y_pred[i], y[i]) d = data(y_pred, y) state = validator.run(d, max_epochs=1, epoch_length=y_pred.shape[0]) assert set(state.metrics.keys()) == set([metric_name]) np_y_pred = np.argmax(y_pred.numpy(), axis=-1).ravel() np_y = y.numpy().ravel() assert state.metrics[metric_name] == approx( compute_true_value_fn(np_y_pred, np_y)) precision_1 = Precision(average=False) precision_2 = Precision(average=False) norm_summed_precision = (precision_1 + precision_2).norm(p=10) def compute_true_norm_summed_precision(y_pred, y): p1 = precision_score(y, y_pred, average=None) p2 = precision_score(y, y_pred, average=None) return np.linalg.norm(p1 + p2, ord=10) _test(norm_summed_precision, "mean summed precision", compute_true_value_fn=compute_true_norm_summed_precision) precision = Precision(average=False) recall = Recall(average=False) sum_precision_recall = (precision + recall).sum() def compute_sum_precision_recall(y_pred, y): p = precision_score(y, y_pred, average=None) r = recall_score(y, y_pred, average=None) return np.sum(p + r) _test(sum_precision_recall, "sum precision recall", compute_true_value_fn=compute_sum_precision_recall) precision = Precision(average=False) recall = Recall(average=False) f1 = (precision * recall * 2 / (precision + recall + 1e-20)).mean() def compute_f1(y_pred, y): f1 = f1_score(y, y_pred, average="macro") return f1 _test(f1, "f1", compute_true_value_fn=compute_f1)
def train(lstm): device = "cpu" train_loader = zip(train_iter, train_logits_iter) test_loader = zip(test_iter, test_logits_iter) optimizer = Adam(lstm.parameters(), 0.001) ce_loss = torch.nn.CrossEntropyLoss() running_avgs = OrderedDict() def step(engine, batch): x, teacher_logits = batch teacher_logits = teacher_logits[0] text = x.text.to(device) aspect = x.aspect.to(device) y_true = x.label.to(device) n_batch = len(aspect) optimizer.zero_grad() y_pred = lstm(text.transpose(0, 1), aspect) distill_loss = distillation_loss(y_pred, y_true, teacher_logits=teacher_logits) distill_loss.backward() optimizer.step() return {'distill_loss': distill_loss.item()} metrics = { 'avg_accuracy': Accuracy(), 'avg_precision': Precision(average=True), 'avg_recall': Recall(average=True) } trainer = Engine(step) train_evaluator = custom_supervised_evaluator(lstm, metrics=metrics, device=device, loss=ce_loss) val_evaluator = custom_supervised_evaluator(lstm, metrics=metrics, device=device, loss=ce_loss) @trainer.on(Events.EPOCH_COMPLETED) def compute_and_display_offline_train_metrics(engine): epoch = engine.state.epoch metrics = train_evaluator.run(train_loader).metrics print( "Training Results - Epoch: {} Accuracy: {:.4f} | Precision: {:.4f} | Recall: {:.4f}" .format(engine.state.epoch, metrics['avg_accuracy'], metrics['avg_precision'], metrics['avg_recall'])) @trainer.on(Events.EPOCH_COMPLETED) def compute_and_display_val_metrics(engine): epoch = engine.state.epoch metrics = val_evaluator.run(test_loader).metrics print( "Validation Results - Epoch: {} Accuracy: {:.4f} | Precision: {:.4f} | Recall: {:.4f}" .format(engine.state.epoch, metrics['avg_accuracy'], metrics['avg_precision'], metrics['avg_recall'])) checkpoint_handler = ModelCheckpoint("./", 'checkpoint', save_interval=3, n_saved=10, require_empty=False, create_dir=True) progress_bar = Progbar(loader=train_loader, metrics=running_avgs) trainer.add_event_handler(event_name=Events.EPOCH_COMPLETED, handler=checkpoint_handler, to_save={'net': lstm}) trainer.add_event_handler(event_name=Events.ITERATION_COMPLETED, handler=progress_bar) trainer.run(train_loader, max_epochs=50)
def test_indexing_metric(): def _test(ignite_metric, sklearn_metic, sklearn_args, index, num_classes=5): y_pred = torch.rand(15, 10, num_classes).float() y = torch.randint(0, num_classes, size=(15, 10)).long() def update_fn(engine, batch): y_pred, y = batch return y_pred, y metrics = { "metric": ignite_metric[index], "metric_wo_index": ignite_metric } validator = Engine(update_fn) for name, metric in metrics.items(): metric.attach(validator, name) def data(y_pred, y): for i in range(y_pred.shape[0]): yield (y_pred[i], y[i]) d = data(y_pred, y) state = validator.run(d, max_epochs=1, epoch_length=y_pred.shape[0]) sklearn_output = sklearn_metic( y.view(-1).numpy(), y_pred.view(-1, num_classes).argmax(dim=1).numpy(), **sklearn_args) assert (state.metrics["metric_wo_index"][index] == state.metrics["metric"]).all() assert np.allclose(state.metrics["metric"].numpy(), sklearn_output) num_classes = 5 labels = list(range(0, num_classes, 2)) _test(Precision(), precision_score, { "labels": labels, "average": None }, index=labels) labels = list(range(num_classes - 1, 0, -2)) _test(Precision(), precision_score, { "labels": labels, "average": None }, index=labels) labels = [1] _test(Precision(), precision_score, { "labels": labels, "average": None }, index=labels) labels = list(range(0, num_classes, 2)) _test(Recall(), recall_score, { "labels": labels, "average": None }, index=labels) labels = list(range(num_classes - 1, 0, -2)) _test(Recall(), recall_score, { "labels": labels, "average": None }, index=labels) labels = [1] _test(Recall(), recall_score, { "labels": labels, "average": None }, index=labels) # np.ix_ is used to allow for a 2D slice of a matrix. This is required to get accurate result from # ConfusionMatrix. ConfusionMatrix must be sliced the same row-wise and column-wise. labels = list(range(0, num_classes, 2)) _test(ConfusionMatrix(num_classes), confusion_matrix, {"labels": labels}, index=np.ix_(labels, labels)) labels = list(range(num_classes - 1, 0, -2)) _test(ConfusionMatrix(num_classes), confusion_matrix, {"labels": labels}, index=np.ix_(labels, labels)) labels = [1] _test(ConfusionMatrix(num_classes), confusion_matrix, {"labels": labels}, index=np.ix_(labels, labels))
def run(loop: Loop): seed_everything(42) setup_cudnn_reproducibility(True, False) train_ds, valid_ds = get_train_test_datasets("data/cifar") model = auto_model(get_model()) train_loader = auto_dataloader( train_ds, batch_size=512, shuffle=True, drop_last=True, num_workers=4, ) valid_loader = auto_dataloader( valid_ds, batch_size=512, num_workers=4, shuffle=False, ) optim = SGD(model.parameters(), lr=0.4, momentum=0.9) scheduler = OneCycleLR(optim, max_lr=1, epochs=NUM_EPOCHS, steps_per_epoch=len(train_loader)) criterion = CrossEntropyLoss() precision = Precision(average=False) recall = Recall(average=False) # Ignite metrics are combinable f1 = (precision * recall * 2 / (precision + recall)).mean() accuracy = Accuracy() # We are attaching metrics to automatically reset loop.attach( # Loop manages train/eval modes, device and requires_grad of attached `nn.Module`s criterion=criterion, # This criterion doesn't have any state or attribute tensors # So it's attachment doesn't introduce any behavior model=model, # Loop saves state of all attached objects having state_dict()/load_state_dict() methods # to checkpoints optimizer=optim, scheduler=scheduler, ) def train(loop: Loop): for _ in loop.iterate_epochs(NUM_EPOCHS): for x, y in loop.iterate_dataloader(train_loader, mode="train"): y_pred_logits = model(x) loss: torch.Tensor = criterion(y_pred_logits, y) loop.backward(loss) # Makes optimizer step and also # zeroes grad after (default) loop.optimizer_step(optim, zero_grad=True) # Here we call scheduler.step() every iteration # because we have one-cycle scheduler # we also can call it after all dataloader loop # if it's som usual scheduler scheduler.step() # Log learning rate. All metrics are written to tensorboard # with specified names # If iteration='auto' (default) its determined based on where the call is # performed. Here it will be batches loop.metrics.log("lr", scheduler.get_last_lr()[0], iteration="auto") # Loop disables gradients and calls Module.eval() inside loop # for all attached modules when mode="valid" (default) for x, y in loop.iterate_dataloader(valid_loader, mode="valid"): y_pred_logits: torch.Tensor = model(x) y_pred = to_onehot(y_pred_logits.argmax(dim=-1), num_classes=10) precision.update((y_pred, y)) recall.update((y_pred, y)) accuracy.update((y_pred, y)) # This metrics will be epoch metrics because they are called outside # dataloader loop # Here we logging metric without resetting it loop.metrics.log("valid/precision", precision.compute().mean()) loop.metrics.log("valid/recall", recall.compute().mean()) # .log() method above accepts values (tensors, floats, np.array's) # .consume() accepts Metric object. It resets it after logging loop.metrics.consume("valid/f1", f1) loop.metrics.consume("valid/accuracy", accuracy) loop.run(train)
seq_len = text[1].cpu().numpy() seq_len[::-1].sort() softmax = nn.Softmax(dim=1) y_pred = classifier(x, seq_len) y_pred = softmax(y_pred) return y_pred.cpu(), y.squeeze().cpu() trainer = Engine(training_update_function) evaluator = create_supervised_evaluator(model=classifier, inference_fn=inference_function, metrics={ "loss": Loss(loss_fn), "acc": CategoricalAccuracy(), "prec": Precision(), "rec": Recall() }) checkpoint = ModelCheckpoint(ARGS.model_dir, "sentiment", save_interval=ARGS.save_interval, n_saved=5, create_dir=True, require_empty=False) loader = ModelLoader(classifier, ARGS.model_dir, "sentiment") model_name = model_config["model"].split(".")[1] # Event handlers trainer.add_event_handler(Events.STARTED, loader, model_name) trainer.add_event_handler(Events.ITERATION_COMPLETED, checkpoint, {model_name: classifier.module}) trainer.add_event_handler(Events.COMPLETED, checkpoint,
def test_multilabel_wrong_inputs(): re = Recall(average=True, is_multilabel=True) with pytest.raises(ValueError): # incompatible shapes re.update((torch.randint(0, 2, size=(10, )), torch.randint(0, 2, size=(10, )).long())) with pytest.raises(ValueError): # incompatible y_pred re.update((torch.rand(10, 5), torch.randint(0, 2, size=(10, 5)).long())) with pytest.raises(ValueError): # incompatible y re.update((torch.randint(0, 5, size=(10, 5, 6)), torch.rand(10))) with pytest.raises(ValueError): # incompatible shapes between two updates re.update((torch.randint(0, 2, size=(20, 5)), torch.randint(0, 2, size=(20, 5)).long())) re.update((torch.randint(0, 2, size=(20, 6)), torch.randint(0, 2, size=(20, 6)).long()))
def train(self, config, **kwargs): """Trains a given model specified in the config file or passed as the --model parameter. All options in the config file can be overwritten as needed by passing --PARAM Options with variable lengths ( e.g., kwargs can be passed by --PARAM '{"PARAM1":VAR1, "PARAM2":VAR2}' :param config: yaml config file :param **kwargs: parameters to overwrite yaml config """ config_parameters = utils.parse_config_or_kwargs(config, **kwargs) outputdir = os.path.join( config_parameters['outputpath'], config_parameters['model'], "{}_{}".format( datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%m'), uuid.uuid1().hex)) # Create base dir Path(outputdir).mkdir(exist_ok=True, parents=True) logger = utils.getfile_outlogger(os.path.join(outputdir, 'train.log')) logger.info("Storing files in {}".format(outputdir)) # utils.pprint_dict utils.pprint_dict(config_parameters, logger.info) logger.info("Running on device {}".format(DEVICE)) labels_df = pd.read_csv(config_parameters['label'], sep='\s+').convert_dtypes() # In case of ave dataset where index is int, we change the # absolute name to relname if not np.all(labels_df['filename'].str.isnumeric()): labels_df.loc[:, 'filename'] = labels_df['filename'].apply( os.path.basename) encoder = utils.train_labelencoder(labels=labels_df['event_labels']) # These labels are useless, only for mode == stratified label_array, _ = utils.encode_labels(labels_df['event_labels'], encoder) if 'cv_label' in config_parameters: cv_df = pd.read_csv(config_parameters['cv_label'], sep='\s+').convert_dtypes() if not np.all(cv_df['filename'].str.isnumeric()): cv_df.loc[:, 'filename'] = cv_df['filename'].apply( os.path.basename) train_df = labels_df logger.info( f"Using CV labels from {config_parameters['cv_label']}") else: train_df, cv_df = utils.split_train_cv( labels_df, y=label_array, **config_parameters['data_args']) if 'cv_data' in config_parameters: cv_data = config_parameters['cv_data'] logger.info(f"Using CV data {config_parameters['cv_data']}") else: cv_data = config_parameters['data'] train_label_array, _ = utils.encode_labels(train_df['event_labels'], encoder) cv_label_array, _ = utils.encode_labels(cv_df['event_labels'], encoder) transform = utils.parse_transforms(config_parameters['transforms']) utils.pprint_dict({'Classes': encoder.classes_}, logger.info, formatter='pretty') torch.save(encoder, os.path.join(outputdir, 'run_encoder.pth')) torch.save(config_parameters, os.path.join(outputdir, 'run_config.pth')) logger.info("Transforms:") utils.pprint_dict(transform, logger.info, formatter='pretty') # For Unbalanced Audioset, this is true if 'sampler' in config_parameters and config_parameters[ 'sampler'] == 'MultiBalancedSampler': # Training sampler that oversamples the dataset to be roughly equally sized # Calcualtes mean over multiple instances, rather useful when number of classes # is large train_sampler = dataset.MultiBalancedSampler( train_label_array, num_samples=1 * train_label_array.shape[0], replacement=True) sampling_kwargs = {"shuffle": False, "sampler": train_sampler} elif 'sampler' in config_parameters and config_parameters[ 'sampler'] == 'MinimumOccupancySampler': # Asserts that each "batch" contains at least one instance train_sampler = dataset.MinimumOccupancySampler( train_label_array, sampling_mode='same') sampling_kwargs = {"shuffle": False, "sampler": train_sampler} else: sampling_kwargs = {"shuffle": True} logger.info("Using Sampler {}".format(sampling_kwargs)) trainloader = dataset.getdataloader( { 'filename': train_df['filename'].values, 'encoded': train_label_array }, config_parameters['data'], transform=transform, batch_size=config_parameters['batch_size'], colname=config_parameters['colname'], num_workers=config_parameters['num_workers'], **sampling_kwargs) cvdataloader = dataset.getdataloader( { 'filename': cv_df['filename'].values, 'encoded': cv_label_array }, cv_data, transform=None, shuffle=False, colname=config_parameters['colname'], batch_size=config_parameters['batch_size'], num_workers=config_parameters['num_workers']) model = getattr(models, config_parameters['model'], 'CRNN')(inputdim=trainloader.dataset.datadim, outputdim=len(encoder.classes_), **config_parameters['model_args']) if 'pretrained' in config_parameters and config_parameters[ 'pretrained'] is not None: models.load_pretrained(model, config_parameters['pretrained'], outputdim=len(encoder.classes_)) logger.info("Loading pretrained model {}".format( config_parameters['pretrained'])) model = model.to(DEVICE) if config_parameters['optimizer'] == 'AdaBound': try: import adabound optimizer = adabound.AdaBound( model.parameters(), **config_parameters['optimizer_args']) except ImportError: config_parameters['optimizer'] = 'Adam' config_parameters['optimizer_args'] = {} else: optimizer = getattr( torch.optim, config_parameters['optimizer'], )(model.parameters(), **config_parameters['optimizer_args']) utils.pprint_dict(optimizer, logger.info, formatter='pretty') utils.pprint_dict(model, logger.info, formatter='pretty') if DEVICE.type != 'cpu' and torch.cuda.device_count() > 1: logger.info("Using {} GPUs!".format(torch.cuda.device_count())) model = torch.nn.DataParallel(model) criterion = getattr(losses, config_parameters['loss'])().to(DEVICE) def _train_batch(_, batch): model.train() with torch.enable_grad(): optimizer.zero_grad() output = self._forward( model, batch) # output is tuple (clip, frame, target) loss = criterion(*output) loss.backward() # Single loss optimizer.step() return loss.item() def _inference(_, batch): model.eval() with torch.no_grad(): return self._forward(model, batch) def thresholded_output_transform(output): # Output is (clip, frame, target) y_pred, _, y = output y_pred = torch.round(y_pred) return y_pred, y precision = Precision(thresholded_output_transform, average=False) recall = Recall(thresholded_output_transform, average=False) f1_score = (precision * recall * 2 / (precision + recall)).mean() metrics = { 'Loss': losses.Loss( criterion), #reimplementation of Loss, supports 3 way loss 'Precision': Precision(thresholded_output_transform), 'Recall': Recall(thresholded_output_transform), 'Accuracy': Accuracy(thresholded_output_transform), 'F1': f1_score, } train_engine = Engine(_train_batch) inference_engine = Engine(_inference) for name, metric in metrics.items(): metric.attach(inference_engine, name) def compute_metrics(engine): inference_engine.run(cvdataloader) results = inference_engine.state.metrics output_str_list = [ "Validation Results - Epoch : {:<5}".format(engine.state.epoch) ] for metric in metrics: output_str_list.append("{} {:<5.2f}".format( metric, results[metric])) logger.info(" ".join(output_str_list)) pbar = ProgressBar(persist=False) pbar.attach(train_engine) if 'itercv' in config_parameters and config_parameters[ 'itercv'] is not None: train_engine.add_event_handler( Events.ITERATION_COMPLETED(every=config_parameters['itercv']), compute_metrics) train_engine.add_event_handler(Events.EPOCH_COMPLETED, compute_metrics) # Default scheduler is using patience=3, factor=0.1 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( optimizer, **config_parameters['scheduler_args']) @inference_engine.on(Events.EPOCH_COMPLETED) def update_reduce_on_plateau(engine): logger.info(f"Scheduling epoch {engine.state.epoch}") val_loss = engine.state.metrics['Loss'] if 'ReduceLROnPlateau' == scheduler.__class__.__name__: scheduler.step(val_loss) else: scheduler.step() early_stop_handler = EarlyStopping( patience=config_parameters['early_stop'], score_function=self._negative_loss, trainer=train_engine) inference_engine.add_event_handler(Events.EPOCH_COMPLETED, early_stop_handler) if config_parameters['save'] == 'everyepoch': checkpoint_handler = ModelCheckpoint(outputdir, 'run', n_saved=5, require_empty=False) train_engine.add_event_handler(Events.EPOCH_COMPLETED, checkpoint_handler, { 'model': model, }) train_engine.add_event_handler( Events.ITERATION_COMPLETED(every=config_parameters['itercv']), checkpoint_handler, { 'model': model, }) else: checkpoint_handler = ModelCheckpoint( outputdir, 'run', n_saved=1, require_empty=False, score_function=self._negative_loss, global_step_transform=global_step_from_engine( train_engine), # Just so that model is saved with epoch... score_name='loss') inference_engine.add_event_handler(Events.EPOCH_COMPLETED, checkpoint_handler, { 'model': model, }) train_engine.run(trainloader, max_epochs=config_parameters['epochs']) return outputdir
def _test_distrib_integration_multilabel(device): from ignite.engine import Engine rank = idist.get_rank() torch.manual_seed(12) def _test(average, n_epochs, metric_device): n_iters = 60 s = 16 n_classes = 7 offset = n_iters * s y_true = torch.randint(0, 2, size=(offset * idist.get_world_size(), n_classes, 6, 8)).to(device) y_preds = torch.randint(0, 2, size=(offset * idist.get_world_size(), n_classes, 6, 8)).to(device) def update(engine, i): return ( y_preds[i * s + rank * offset:(i + 1) * s + rank * offset, ...], y_true[i * s + rank * offset:(i + 1) * s + rank * offset, ...], ) engine = Engine(update) re = Recall(average=average, is_multilabel=True, device=metric_device) re.attach(engine, "re") data = list(range(n_iters)) engine.run(data=data, max_epochs=n_epochs) assert "re" in engine.state.metrics res = engine.state.metrics["re"] res2 = re.compute() if isinstance(res, torch.Tensor): res = res.cpu().numpy() res2 = res2.cpu().numpy() assert (res == res2).all() else: assert res == res2 with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) true_res = recall_score(to_numpy_multilabel(y_true), to_numpy_multilabel(y_preds), average="samples" if average else None) assert pytest.approx(res) == true_res metric_devices = ["cpu"] if device.type != "xla": metric_devices.append(idist.device()) for _ in range(2): for metric_device in metric_devices: _test(average=True, n_epochs=1, metric_device=metric_device) _test(average=True, n_epochs=2, metric_device=metric_device) if idist.get_world_size() > 1: with pytest.warns( RuntimeWarning, match= "Precision/Recall metrics do not work in distributed setting when " "average=False and is_multilabel=True", ): re = Recall(average=False, is_multilabel=True) y_pred = torch.randint(0, 2, size=(4, 3, 6, 8)) y = torch.randint(0, 2, size=(4, 3, 6, 8)).long() re.update((y_pred, y)) re_compute1 = re.compute() re_compute2 = re.compute() assert len(re_compute1) == 4 * 6 * 8 assert (re_compute1 == re_compute2).all()
def _test(average): re = Recall(average=average) y_pred = torch.rand(20, 6) y = torch.randint(0, 6, size=(20, )).long() re.update((y_pred, y)) num_classes = y_pred.shape[1] np_y_pred = y_pred.argmax(dim=1).numpy().ravel() np_y = y.numpy().ravel() assert re._type == "multiclass" assert isinstance(re.compute(), float if average else torch.Tensor) re_compute = re.compute() if average else re.compute().numpy() sk_average_parameter = "macro" if average else None with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) sk_compute = recall_score( np_y, np_y_pred, labels=range(0, num_classes), average=sk_average_parameter, ) assert sk_compute == pytest.approx(re_compute) re.reset() y_pred = torch.rand(10, 4) y = torch.randint(0, 4, size=(10, )).long() re.update((y_pred, y)) num_classes = y_pred.shape[1] np_y_pred = y_pred.argmax(dim=1).numpy().ravel() np_y = y.numpy().ravel() assert re._type == "multiclass" assert isinstance(re.compute(), float if average else torch.Tensor) re_compute = re.compute() if average else re.compute().numpy() sk_average_parameter = "macro" if average else None with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) sk_compute = recall_score( np_y, np_y_pred, labels=range(0, num_classes), average=sk_average_parameter, ) assert sk_compute == pytest.approx(re_compute) # 2-classes re.reset() y_pred = torch.rand(10, 2) y = torch.randint(0, 2, size=(10, )).long() re.update((y_pred, y)) num_classes = y_pred.shape[1] np_y_pred = y_pred.argmax(dim=1).numpy().ravel() np_y = y.numpy().ravel() assert re._type == "multiclass" assert isinstance(re.compute(), float if average else torch.Tensor) re_compute = re.compute() if average else re.compute().numpy() sk_average_parameter = "macro" if average else None with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) sk_compute = recall_score( np_y, np_y_pred, labels=range(0, num_classes), average=sk_average_parameter, ) assert sk_compute == pytest.approx(re_compute) # Batched Updates re.reset() y_pred = torch.rand(100, 3) y = torch.randint(0, 3, size=(100, )).long() batch_size = 16 n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size re.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) num_classes = y_pred.shape[1] np_y = y.numpy().ravel() np_y_pred = y_pred.argmax(dim=1).numpy().ravel() assert re._type == "multiclass" assert isinstance(re.compute(), float if average else torch.Tensor) re_compute = re.compute() if average else re.compute().numpy() sk_average_parameter = "macro" if average else None with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) sk_compute = recall_score( np_y, np_y_pred, labels=range(0, num_classes), average=sk_average_parameter, ) assert sk_compute == pytest.approx(re_compute)
def _test(average): re = Recall(average=average) y_pred = torch.randint(0, 2, size=(10, 12, 10)) y = torch.randint(0, 2, size=(10, 12, 10)).type(torch.LongTensor) re.update((y_pred, y)) np_y = y.numpy().ravel() np_y_pred = y_pred.numpy().ravel() assert re._type == 'binary' assert isinstance(re.compute(), float if average else torch.Tensor) re_compute = re.compute() if average else re.compute().numpy() assert recall_score(np_y, np_y_pred, average='binary') == pytest.approx(re_compute) re.reset() y_pred = torch.randint(0, 2, size=(10, 1, 12, 10)) y = torch.randint(0, 2, size=(10, 1, 12, 10)).type(torch.LongTensor) re.update((y_pred, y)) np_y = y.numpy().ravel() np_y_pred = y_pred.numpy().ravel() assert re._type == 'binary' assert isinstance(re.compute(), float if average else torch.Tensor) re_compute = re.compute() if average else re.compute().numpy() assert recall_score(np_y, np_y_pred, average='binary') == pytest.approx(re_compute) re = Recall(average=average) # Batched Updates re.reset() y_pred = torch.randint(0, 2, size=(100, 12, 10)) y = torch.randint(0, 2, size=(100, 1, 12, 10)).type(torch.LongTensor) batch_size = 16 n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size re.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) np_y = y.numpy().ravel() np_y_pred = y_pred.numpy().ravel() assert re._type == 'binary' assert isinstance(re.compute(), float if average else torch.Tensor) re_compute = re.compute() if average else re.compute().numpy() assert recall_score(np_y, np_y_pred, average='binary') == pytest.approx(re_compute)
# F1 = (precision * recall * 2 / (precision + recall)).mean() tqdm.write( "Validation Results - Epoch: {} Avg accuracy: {:.2f} Avg loss: {:.2f}" .format(engine.state.epoch, avg_accuracy, avg_loss)) pbar.n = pbar.last_print_n = 0 trainer.run(train_loader, max_epochs=args.num_epochs) pbar.close() tester = create_supervised_evaluator(model, metrics={ 'accuracy': Accuracy(), 'loss': Loss(criterion), 'pre': Precision(average=True), 'recall': Recall(average=False) }, device=device) tester.run(test_loader) metrics = tester.state.metrics test_accuracy = metrics['accuracy'] test_loss = metrics['loss'] print("Precision", metrics['pre']) print("Recall", metrics['recall']) print("Test Results - Avg accuracy: {:.2f} Avg loss: {:.2f}".format( test_accuracy, test_loss)) stats = { 'train_accuracy': train_accuracy, 'train_loss': train_loss, 'val_accuracy': val_accuracy, 'val_loss': val_loss,
def _test(average): re = Recall(average=average) y_pred = torch.rand(10, 5, 8) y = torch.randint(0, 5, size=(10, 8)).type(torch.LongTensor) re.update((y_pred, y)) num_classes = y_pred.shape[1] np_y_pred = y_pred.numpy().argmax(axis=1).ravel() np_y = y.numpy().ravel() assert re._type == 'multiclass' assert isinstance(re.compute(), float if average else torch.Tensor) re_compute = re.compute() if average else re.compute().numpy() sk_average_parameter = 'macro' if average else None with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert recall_score( np_y, np_y_pred, average=sk_average_parameter) == pytest.approx(re_compute) re.reset() y_pred = torch.rand(15, 10, 8) y = torch.randint(0, 10, size=(15, 8)).type(torch.LongTensor) re.update((y_pred, y)) num_classes = y_pred.shape[1] np_y_pred = y_pred.numpy().argmax(axis=1).ravel() np_y = y.numpy().ravel() assert re._type == 'multiclass' assert isinstance(re.compute(), float if average else torch.Tensor) re_compute = re.compute() if average else re.compute().numpy() sk_average_parameter = 'macro' if average else None with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) sk_compute = recall_score(np_y, np_y_pred, labels=range(0, num_classes), average=sk_average_parameter) assert sk_compute == pytest.approx(re_compute) # Batched Updates re.reset() y_pred = torch.rand(100, 8, 12) y = torch.randint(0, 8, size=(100, 12)).type(torch.LongTensor) batch_size = 16 n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size re.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) num_classes = y_pred.shape[1] np_y = y.numpy().ravel() np_y_pred = y_pred.numpy().argmax(axis=1).ravel() assert re._type == 'multiclass' assert isinstance(re.compute(), float if average else torch.Tensor) re_compute = re.compute() if average else re.compute().numpy() sk_average_parameter = 'macro' if average else None with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) sk_compute = recall_score(np_y, np_y_pred, labels=range(0, num_classes), average=sk_average_parameter) assert sk_compute == pytest.approx(re_compute)
def test_multilabel_input(average): re = Recall(average=average, is_multilabel=True) assert re._updated is False def _test(y_pred, y, batch_size): re.reset() assert re._updated is False if batch_size > 1: n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size re.update( (y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) else: re.update((y_pred, y)) np_y_pred = to_numpy_multilabel(y_pred) np_y = to_numpy_multilabel(y) assert re._type == "multilabel" assert re._updated is True re_compute = re.compute() if average else re.compute().mean().item() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UndefinedMetricWarning) assert recall_score(np_y, np_y_pred, average="samples") == pytest.approx(re_compute) re1 = Recall(is_multilabel=True, average=True) re2 = Recall(is_multilabel=True, average=False) assert re1._updated is False assert re2._updated is False re1.update((y_pred, y)) re2.update((y_pred, y)) assert re1._updated is True assert re2._updated is True assert re1.compute() == pytest.approx(re2.compute().mean().item()) assert re1._updated is True assert re2._updated is True def get_test_cases(): test_cases = [ # Multilabel input data of shape (N, C) (torch.randint(0, 2, size=(10, 5)), torch.randint(0, 2, size=(10, 5)), 1), (torch.randint(0, 2, size=(10, 4)), torch.randint(0, 2, size=(10, 4)), 1), # updated batches (torch.randint(0, 2, size=(50, 5)), torch.randint(0, 2, size=(50, 5)), 16), (torch.randint(0, 2, size=(50, 4)), torch.randint(0, 2, size=(50, 4)), 16), # Multilabel input data of shape (N, H, W) (torch.randint(0, 2, size=(10, 5, 10)), torch.randint(0, 2, size=(10, 5, 10)), 1), (torch.randint(0, 2, size=(10, 4, 10)), torch.randint(0, 2, size=(10, 4, 10)), 1), # updated batches (torch.randint(0, 2, size=(50, 5, 10)), torch.randint(0, 2, size=(50, 5, 10)), 16), (torch.randint(0, 2, size=(50, 4, 10)), torch.randint(0, 2, size=(50, 4, 10)), 16), # Multilabel input data of shape (N, C, H, W, ...) (torch.randint(0, 2, size=(10, 5, 18, 16)), torch.randint(0, 2, size=(10, 5, 18, 16)), 1), (torch.randint(0, 2, size=(10, 4, 20, 23)), torch.randint(0, 2, size=(10, 4, 20, 23)), 1), # updated batches (torch.randint(0, 2, size=(50, 5, 18, 16)), torch.randint(0, 2, size=(50, 5, 18, 16)), 16), (torch.randint(0, 2, size=(50, 4, 20, 23)), torch.randint(0, 2, size=(50, 4, 20, 23)), 16), # Corner case with all zeros predictions (torch.zeros(size=(10, 5)), torch.randint(0, 2, size=(10, 5)), 1), (torch.zeros(size=(10, 4)), torch.randint(0, 2, size=(10, 4)), 1), ] return test_cases for _ in range(5): # check multiple random inputs as random exact occurencies are rare test_cases = get_test_cases() for y_pred, y, batch_size in test_cases: _test(y_pred, y, batch_size)