def test_integration_ap_score_with_activated_output_transform(): np.random.seed(1) size = 100 np_y_pred = np.random.rand(size, 1) np_y_pred_softmax = torch.softmax(torch.from_numpy(np_y_pred), dim=1).numpy() np_y = np.zeros((size, ), dtype=np.long) np_y[size // 2:] = 1 np.random.shuffle(np_y) np_ap = average_precision_score(np_y, np_y_pred_softmax) batch_size = 10 def update_fn(engine, batch): idx = (engine.state.iteration - 1) * batch_size y_true_batch = np_y[idx:idx + batch_size] y_pred_batch = np_y_pred[idx:idx + batch_size] return idx, torch.from_numpy(y_pred_batch), torch.from_numpy( y_true_batch) engine = Engine(update_fn) ap_metric = AveragePrecision( output_transform=lambda x: (torch.softmax(x[1], dim=1), x[2])) ap_metric.attach(engine, 'ap') data = list(range(size // batch_size)) ap = engine.run(data, max_epochs=1).metrics['ap'] assert ap == np_ap
def test_no_update(): ap = AveragePrecision() with pytest.raises( NotComputableError, match= r"EpochMetric must have at least one example before it can be computed" ): ap.compute()
def _test(y_pred, y, n_iters, metric_device): metric_device = torch.device(metric_device) ap = AveragePrecision(device=metric_device) torch.manual_seed(10 + rank) ap.reset() ap.update((y_pred, y)) if n_iters > 1: batch_size = y.shape[0] // n_iters + 1 for i in range(n_iters): idx = i * batch_size ap.update( (y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) # gather y_pred, y y_pred = idist.all_gather(y_pred) y = idist.all_gather(y) np_y = y.cpu().numpy() np_y_pred = y_pred.cpu().numpy() res = ap.compute() assert isinstance(res, float) assert average_precision_score(np_y, np_y_pred) == pytest.approx(res)
def test_ap_score_2(): np.random.seed(1) size = 100 np_y_pred = np.random.rand(size, 1) np_y = np.zeros((size, ), dtype=np.long) np_y[size // 2:] = 1 np.random.shuffle(np_y) np_ap = average_precision_score(np_y, np_y_pred) ap_metric = AveragePrecision() y_pred = torch.from_numpy(np_y_pred) y = torch.from_numpy(np_y) ap_metric.reset() n_iters = 10 batch_size = size // n_iters for i in range(n_iters): idx = i * batch_size ap_metric.update( (y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) ap = ap_metric.compute() assert ap == np_ap
def _test(y_preds, y_true, n_epochs, metric_device, update_fn): metric_device = torch.device(metric_device) engine = Engine(update_fn) ap = AveragePrecision(device=metric_device) ap.attach(engine, "ap") data = list(range(n_iters)) engine.run(data=data, max_epochs=n_epochs) assert "ap" in engine.state.metrics res = engine.state.metrics["ap"] true_res = average_precision_score(y_true.cpu().numpy(), y_preds.cpu().numpy()) assert pytest.approx(res) == true_res
def test_check_shape(): ap = AveragePrecision() with pytest.raises(ValueError, match=r"Predictions should be of shape"): ap._check_shape((torch.tensor(0), torch.tensor(0))) with pytest.raises(ValueError, match=r"Predictions should be of shape"): ap._check_shape((torch.rand(4, 3, 1), torch.rand(4, 3))) with pytest.raises(ValueError, match=r"Targets should be of shape"): ap._check_shape((torch.rand(4, 3), torch.rand(4, 3, 1)))
def test_binary_and_multilabel_inputs(): ap = AveragePrecision() def _test(y_pred, y, batch_size): ap.reset() if batch_size > 1: n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size ap.update( (y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) else: ap.update((y_pred, y)) np_y = y.numpy() np_y_pred = y_pred.numpy() res = ap.compute() assert isinstance(res, float) assert average_precision_score(np_y, np_y_pred) == pytest.approx(res) def get_test_cases(): test_cases = [ # Binary input data of shape (N,) or (N, 1) (torch.randint(0, 2, size=(50, )).long(), torch.randint(0, 2, size=(50, )).long(), 1), (torch.randint(0, 2, size=(50, 1)).long(), torch.randint(0, 2, size=(50, 1)).long(), 1), # updated batches (torch.randint(0, 2, size=(50, )).long(), torch.randint(0, 2, size=(50, )).long(), 16), (torch.randint(0, 2, size=(50, 1)).long(), torch.randint(0, 2, size=(50, 1)).long(), 16), # Binary input data of shape (N, L) (torch.randint(0, 2, size=(50, 4)).long(), torch.randint(0, 2, size=(50, 4)).long(), 1), (torch.randint(0, 2, size=(50, 7)).long(), torch.randint(0, 2, size=(50, 7)).long(), 1), # updated batches (torch.randint(0, 2, size=(50, 4)).long(), torch.randint(0, 2, size=(50, 4)).long(), 16), (torch.randint(0, 2, size=(50, 7)).long(), torch.randint(0, 2, size=(50, 7)).long(), 16), ] return test_cases for _ in range(5): # check multiple random inputs as random exact occurencies are rare test_cases = get_test_cases() for y_pred, y, batch_size in test_cases: _test(y_pred, y, batch_size)
def _test(n_epochs, metric_device): metric_device = torch.device(metric_device) n_iters = 80 s = 16 n_classes = 2 offset = n_iters * s y_true = torch.randint(0, n_classes, size=(offset * idist.get_world_size(), 10)).to(device) y_preds = torch.randint(0, n_classes, size=(offset * idist.get_world_size(), 10)).to(device) def update(engine, i): return ( y_preds[i * s + rank * offset:(i + 1) * s + rank * offset, :], y_true[i * s + rank * offset:(i + 1) * s + rank * offset, :], ) engine = Engine(update) ap = AveragePrecision(device=metric_device) ap.attach(engine, "ap") data = list(range(n_iters)) engine.run(data=data, max_epochs=n_epochs) assert "ap" in engine.state.metrics res = engine.state.metrics["ap"] if isinstance(res, torch.Tensor): res = res.cpu().numpy() true_res = average_precision_score(y_true.cpu().numpy(), y_preds.cpu().numpy()) assert pytest.approx(res) == true_res
def _test(y_pred, y, batch_size): def update_fn(engine, batch): idx = (engine.state.iteration - 1) * batch_size y_true_batch = np_y[idx:idx + batch_size] y_pred_batch = np_y_pred[idx:idx + batch_size] return idx, torch.from_numpy(y_pred_batch), torch.from_numpy( y_true_batch) engine = Engine(update_fn) ap_metric = AveragePrecision(output_transform=lambda x: (x[1], x[2])) ap_metric.attach(engine, "ap") np_y = y.numpy() np_y_pred = y_pred.numpy() np_ap = average_precision_score(np_y, np_y_pred) data = list(range(y_pred.shape[0] // batch_size)) ap = engine.run(data, max_epochs=1).metrics["ap"] assert isinstance(ap, float) assert np_ap == pytest.approx(ap)
def test_binary_input_N(): ap = AveragePrecision() def _test(y_pred, y, n_iters): ap.reset() ap.update((y_pred, y)) np_y = y.numpy() np_y_pred = y_pred.numpy() if n_iters > 1: batch_size = y.shape[0] // n_iters + 1 for i in range(n_iters): idx = i * batch_size ap.update( (y_pred[idx:idx + batch_size], y[idx:idx + batch_size])) res = ap.compute() assert isinstance(res, float) assert average_precision_score(np_y, np_y_pred) == pytest.approx(res) def get_test_cases(): test_cases = [ (torch.randint(0, 2, size=(10, )).long(), torch.randint(0, 2, size=(10, )).long(), 1), (torch.randint(0, 2, size=(100, )).long(), torch.randint(0, 2, size=(100, )).long(), 1), (torch.randint(0, 2, size=(10, 1)).long(), torch.randint(0, 2, size=(10, 1)).long(), 1), (torch.randint(0, 2, size=(100, 1)).long(), torch.randint(0, 2, size=(100, 1)).long(), 1), # updated batches (torch.randint(0, 2, size=(10, )).long(), torch.randint(0, 2, size=(10, )).long(), 16), (torch.randint(0, 2, size=(100, )).long(), torch.randint(0, 2, size=(100, )).long(), 16), (torch.randint(0, 2, size=(10, 1)).long(), torch.randint(0, 2, size=(10, 1)).long(), 16), (torch.randint(0, 2, size=(100, 1)).long(), torch.randint(0, 2, size=(100, 1)).long(), 16), ] return test_cases for _ in range(10): # check multiple random inputs as random exact occurencies are rare test_cases = get_test_cases() for y_pred, y, n_iters in test_cases: _test(y_pred, y, n_iters)
def test_ap_score(): size = 100 np_y_pred = np.random.rand(size, 5) np_y = np.random.randint(0, 2, size=(size, 5), dtype=np.long) np_ap = average_precision_score(np_y, np_y_pred) ap_metric = AveragePrecision() y_pred = torch.from_numpy(np_y_pred) y = torch.from_numpy(np_y) ap_metric.reset() ap_metric.update((y_pred, y)) ap = ap_metric.compute() assert ap == np_ap
def test_ap_score_with_activation(): size = 100 np_y_pred = np.random.rand(size, 5) np_y_pred_softmax = torch.softmax(torch.from_numpy(np_y_pred), dim=1).numpy() np_y = np.random.randint(0, 2, size=(size, 5), dtype=np.long) np_ap = average_precision_score(np_y, np_y_pred_softmax) ap_metric = AveragePrecision(activation=torch.nn.Softmax(dim=1)) y_pred = torch.from_numpy(np_y_pred) y = torch.from_numpy(np_y) ap_metric.reset() ap_metric.update((y_pred, y)) ap = ap_metric.compute() assert ap == np_ap
def run(): ### Create tensorboardx writer writer = create_summary_writer(model, train_loader, log_dir) print("Writer created") ### Create trainer, evaulator trainer = Engine(train_batch) evaluator = Engine(eval_fn) #### Attach evaluation metrics Accuracy(output_transform=thresholded_output_transform).attach( evaluator, 'accuracy') Precision(output_transform=thresholded_output_transform).attach( evaluator, 'precision') Recall(output_transform=thresholded_output_transform).attach( evaluator, 'recall') AveragePrecision().attach(evaluator, 'AP') Loss(criterion).attach(evaluator, 'loss') #### tqdm settings if USE_TQDM: desc = "Epoch[{}] Iteration[{}/{}] Loss: {:.2f}" pbar = tqdm(initial=0, leave=False, total=len(train_loader), desc=desc.format(0, 0, 0, 0)) ### Attach on events @trainer.on(Events.ITERATION_COMPLETED) def log_training_loss(engine): iter = (engine.state.iteration - 1) % len(train_loader) + 1 if iter % log_interval == 0: if USE_TQDM: pbar.desc = desc.format(engine.state.epoch, iter, len(train_loader), engine.state.output) pbar.update(log_interval) else: print("Epoch[{}] Iteration[{}/{}] Loss: {:.2f}".format( engine.state.epoch, iter, len(train_loader), engine.state.output)) writer.add_scalar("training/loss", engine.state.output, engine.state.iteration) @trainer.on(Events.EPOCH_COMPLETED) def log_training_results(engine): evaluator.run(train_loader) metrics = evaluator.state.metrics avg_accuracy = metrics['accuracy'] avg_loss = metrics['loss'] prec = metrics['precision'] recall = metrics['recall'] ap = metrics['AP'] print( "Training Results - Epoch: {} Avg accuracy: {:.2f} Avg loss: {:.2f} Precision: {:.2f} Recall: {:.2f} APR: {:.2f}" .format(engine.state.epoch, avg_accuracy, avg_loss, prec, recall, ap)) writer.add_scalar("training/avg_loss", avg_loss, engine.state.epoch) writer.add_scalar("training/avg_accuracy", avg_accuracy, engine.state.epoch) writer.add_scalar("training/precision", prec, engine.state.epoch) writer.add_scalar("training/recall", recall, engine.state.epoch) writer.add_scalar("training/avg precision", ap, engine.state.epoch) @trainer.on(Events.EPOCH_COMPLETED) def log_val_results(engine): evaluator.run(val_loader) metrics = evaluator.state.metrics avg_accuracy = metrics['accuracy'] avg_loss = metrics['loss'] prec = metrics['precision'] recall = metrics['recall'] ap = metrics['AP'] print( "Training Results - Epoch: {} Avg accuracy: {:.2f} Avg loss: {:.2f} Precision: {:.2f} Recall: {:.2f} APR: {:.2f}" .format(engine.state.epoch, avg_accuracy, avg_loss, prec, recall, ap)) writer.add_scalar("validation/avg_loss", avg_loss, engine.state.epoch) writer.add_scalar("validation/avg_accuracy", avg_accuracy, engine.state.epoch) writer.add_scalar("v[{}] Iteration[{}/{}] Loss:alidation/precision", prec, engine.state.epoch) writer.add_scalar("validation/recall", recall, engine.state.epoch) writer.add_scalar("validation/avg precision", ap, engine.state.epoch) #### Run the joint print("Training...") trainer.run(train_loader, max_epochs=epochs) writer.close()
def train(epochs: int, model: nn.Module, train_loader: DataLoader, valid_loader: DataLoader, criterion: Callable, device: str, lr: float, patience: int, lr_decay: float, lr_scheduler: str, lr_scheduler_kwargs: Dict[str, Any]): model.to(torch.device(device)) optimizer = optim.Adam( [param for param in model.parameters() if param.requires_grad], lr=lr) trainer = create_supervised_trainer(model, optimizer, criterion, device=device) scheduler = LRScheduler( getattr(optim.lr_scheduler, lr_scheduler)(optimizer, **lr_scheduler_kwargs)) trainer.add_event_handler(Events.ITERATION_COMPLETED, scheduler) pbar = ProgressBar(False) pbar.attach(trainer) train_evaluator = create_supervised_evaluator( model, metrics={ 'ACC': Accuracy(discreted_output_transform), 'BCE': Loss(criterion), 'AP': AveragePrecision(probability_output_transform) }, device=device) valid_evaluator = create_supervised_evaluator( model, metrics={ 'ACC': Accuracy(discreted_output_transform), 'BCE': Loss(criterion), 'AP': AveragePrecision(probability_output_transform) }, device=device) history = { col: list() for col in [ 'epoch', 'elapsed time', 'iterations', 'lr', 'train BCE', 'valid BCE', 'train ACC', 'valid ACC', 'train AP', 'valid AP' ] } @trainer.on(Events.EPOCH_COMPLETED) def log_training_results(engine): train_evaluator.run(train_loader) history['train BCE'] += [train_evaluator.state.metrics['BCE']] history['train ACC'] += [train_evaluator.state.metrics['ACC']] history['train AP'] += [train_evaluator.state.metrics['AP']] @trainer.on(Events.EPOCH_COMPLETED) def log_validation_results(engine): valid_evaluator.run(valid_loader) history['epoch'] += [valid_evaluator.state.epoch] history['iterations'] += [valid_evaluator.state.epoch_length] history['elapsed time'] += [ 0 if len(history['elapsed time']) == 0 else history['elapsed time'][-1] + valid_evaluator.state.times['COMPLETED'] ] history['lr'] += [scheduler.get_param()] history['valid BCE'] += [valid_evaluator.state.metrics['BCE']] history['valid ACC'] += [valid_evaluator.state.metrics['ACC']] history['valid AP'] += [valid_evaluator.state.metrics['AP']] @trainer.on(Events.EPOCH_COMPLETED) def log_progress_bar(engine): pbar.log_message( f"train BCE: {history['train BCE'][-1]:.2f} " \ + f"train ACC: {history['train ACC'][-1]:.2f} " \ + f"train AP: {history['train AP'][-1]:.2f} " \ + f"valid BCE: {history['valid BCE'][-1]:.2f} " \ + f"valid ACC: {history['valid ACC'][-1]:.2f} " \ + f"valid AP: {history['valid AP'][-1]:.2f}" ) # Early stopping handler = EarlyStopping(patience=patience, score_function=score_function, trainer=trainer) valid_evaluator.add_event_handler(Events.EPOCH_COMPLETED, handler) trainer.run(train_loader, max_epochs=epochs) return pd.DataFrame(history)
def test_input_types(): ap = AveragePrecision() ap.reset() output1 = (torch.rand(4, 3), torch.randint(0, 2, size=(4, 3), dtype=torch.long)) ap.update(output1) with pytest.raises( ValueError, match= r"Incoherent types between input y_pred and stored predictions"): ap.update((torch.randint(0, 5, size=(4, 3)), torch.randint(0, 2, size=(4, 3)))) with pytest.raises( ValueError, match=r"Incoherent types between input y and stored targets"): ap.update((torch.rand(4, 3), torch.randint(0, 2, size=(4, 3)).to(torch.int32))) with pytest.raises( ValueError, match= r"Incoherent types between input y_pred and stored predictions"): ap.update((torch.randint(0, 2, size=(10, )).long(), torch.randint(0, 2, size=(10, 5)).long()))
def test_no_sklearn(mock_no_sklearn): with pytest.raises( RuntimeError, match=r"This contrib module requires sklearn to be installed."): AveragePrecision()
def train(config): model_suite.logging.setup_loggers(config) device = 'cpu' if torch.cuda.is_available(): device = 'cuda' logger.info(f'Device {device} will be used') data_df = read_data() train_ds, val_ds, test_ds = get_datasets(data_df) train_loader, val_loader = get_data_loaders( train_ds, val_ds, train_batch_size=config.train_batch_size, val_batch_size=config.val_batch_size) writer = SummaryWriter(log_dir=f'{config.model_dir}/logs') n_features = train_loader.dataset[0][0].shape[0] model = get_model(model_name=config.model, n_features=n_features) loss = torch.nn.BCEWithLogitsLoss() optimizer = Adam(model.parameters(), lr=config.learning_rate, weight_decay=config.weight_decay) #optimizer = SGD(model.parameters(), lr=config.learning_rate, weight_decay=config.weight_decay, momentum=config.momentum) trainer = create_supervised_trainer(model, optimizer, loss, device=device) evaluator = create_supervised_evaluator(model, metrics={ 'loss': Loss(loss), 'roc': ROC_AUC(), 'accuracy': Accuracy(), 'precision': AveragePrecision() }, device=device) @trainer.on(Events.EPOCH_COMPLETED) def log_training_results(engine): evaluator.run(train_loader) metrics = evaluator.state.metrics avg_loss = metrics['loss'] avg_roc = metrics['roc'] # avg_accuracy = metrics['accuracy'] # avg_precision = metrics['precision'] logger.info( f'Training results - Epoch: {engine.state.epoch} Avg loss: {avg_loss} ROC: {avg_roc}' ) writer.add_scalar("training/avg_loss", avg_loss, engine.state.epoch) writer.add_scalar("training/avg_roc", avg_roc, engine.state.epoch) # writer.add_scalar("training/avg_accuracy", avg_accuracy, engine.state.epoch) # writer.add_scalar("training/avg_precision", avg_precision, engine.state.epoch) @trainer.on(Events.EPOCH_COMPLETED) def log_validation_results(engine): evaluator.run(val_loader) metrics = evaluator.state.metrics avg_loss = metrics['loss'] avg_roc = metrics['roc'] # avg_accuracy = metrics['accuracy'] # avg_precision = metrics['precision'] logger.info( f'Validation results - Epoch: {engine.state.epoch} Avg loss: {avg_loss} ROC: {avg_roc}' ) writer.add_scalar("valdation/avg_loss", avg_loss, engine.state.epoch) writer.add_scalar("valdation/avg_roc", avg_roc, engine.state.epoch) # writer.add_scalar("valdation/avg_accuracy", avg_accuracy, engine.state.epoch) # writer.add_scalar("valdation/avg_precision", avg_precision, engine.state.epoch) trainer.run(train_loader, max_epochs=config.n_epochs) writer.close()
cycle_size=len(train_loader) * cfg.n_epochs, start_value_mult=0, end_value_mult=0), warmup_start_value=0.0, warmup_end_value=cfg.lr, warmup_duration=len(train_loader))) evaluator = create_supervised_evaluator( model, metrics={ 'loss': Loss(loss), 'acc_smpl': Accuracy(threshold_output, is_multilabel=True), 'p': Precision(threshold_output, average=True), 'r': Recall(threshold_output, average=True), 'f1': Fbeta(1.0, output_transform=threshold_output), 'ap': AveragePrecision(output_transform=activate_output) }, device=device) model_checkpoint = ModelCheckpoint( dirname=wandb.run.dir, filename_prefix='best', require_empty=False, score_function=lambda e: e.state.metrics['ap'], global_step_transform=global_step_from_engine(trainer)) evaluator.add_event_handler(Events.COMPLETED, model_checkpoint, {'model': model}) @trainer.on(Events.EPOCH_COMPLETED) def validate(trainer): evaluator.run(val_loader)