def test_integration_ap_score_with_activated_output_transform():
np.random.seed(1)
size = 100
np_y_pred = np.random.rand(size, 1)
np_y_pred_softmax = torch.softmax(torch.from_numpy(np_y_pred),
dim=1).numpy()
np_y = np.zeros((size, ), dtype=np.long)
np_y[size // 2:] = 1
np.random.shuffle(np_y)
np_ap = average_precision_score(np_y, np_y_pred_softmax)
batch_size = 10
def update_fn(engine, batch):
idx = (engine.state.iteration - 1) * batch_size
y_true_batch = np_y[idx:idx + batch_size]
y_pred_batch = np_y_pred[idx:idx + batch_size]
return idx, torch.from_numpy(y_pred_batch), torch.from_numpy(
y_true_batch)
engine = Engine(update_fn)
ap_metric = AveragePrecision(
output_transform=lambda x: (torch.softmax(x[1], dim=1), x[2]))
ap_metric.attach(engine, 'ap')
data = list(range(size // batch_size))
ap = engine.run(data, max_epochs=1).metrics['ap']
assert ap == np_ap
def test_no_update():
ap = AveragePrecision()
with pytest.raises(
NotComputableError,
match=
r"EpochMetric must have at least one example before it can be computed"
):
ap.compute()
def _test(y_pred, y, n_iters, metric_device):
metric_device = torch.device(metric_device)
ap = AveragePrecision(device=metric_device)
torch.manual_seed(10 + rank)
ap.reset()
ap.update((y_pred, y))
if n_iters > 1:
batch_size = y.shape[0] // n_iters + 1
for i in range(n_iters):
idx = i * batch_size
ap.update(
(y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
# gather y_pred, y
y_pred = idist.all_gather(y_pred)
y = idist.all_gather(y)
np_y = y.cpu().numpy()
np_y_pred = y_pred.cpu().numpy()
res = ap.compute()
assert isinstance(res, float)
assert average_precision_score(np_y, np_y_pred) == pytest.approx(res)
def test_ap_score_2():
np.random.seed(1)
size = 100
np_y_pred = np.random.rand(size, 1)
np_y = np.zeros((size, ), dtype=np.long)
np_y[size // 2:] = 1
np.random.shuffle(np_y)
np_ap = average_precision_score(np_y, np_y_pred)
ap_metric = AveragePrecision()
y_pred = torch.from_numpy(np_y_pred)
y = torch.from_numpy(np_y)
ap_metric.reset()
n_iters = 10
batch_size = size // n_iters
for i in range(n_iters):
idx = i * batch_size
ap_metric.update(
(y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
ap = ap_metric.compute()
assert ap == np_ap
def _test(y_preds, y_true, n_epochs, metric_device, update_fn):
metric_device = torch.device(metric_device)
engine = Engine(update_fn)
ap = AveragePrecision(device=metric_device)
ap.attach(engine, "ap")
data = list(range(n_iters))
engine.run(data=data, max_epochs=n_epochs)
assert "ap" in engine.state.metrics
res = engine.state.metrics["ap"]
true_res = average_precision_score(y_true.cpu().numpy(),
y_preds.cpu().numpy())
assert pytest.approx(res) == true_res
def test_check_shape():
ap = AveragePrecision()
with pytest.raises(ValueError, match=r"Predictions should be of shape"):
ap._check_shape((torch.tensor(0), torch.tensor(0)))
with pytest.raises(ValueError, match=r"Predictions should be of shape"):
ap._check_shape((torch.rand(4, 3, 1), torch.rand(4, 3)))
with pytest.raises(ValueError, match=r"Targets should be of shape"):
ap._check_shape((torch.rand(4, 3), torch.rand(4, 3, 1)))
def test_binary_and_multilabel_inputs():
ap = AveragePrecision()
def _test(y_pred, y, batch_size):
ap.reset()
if batch_size > 1:
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
ap.update(
(y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
else:
ap.update((y_pred, y))
np_y = y.numpy()
np_y_pred = y_pred.numpy()
res = ap.compute()
assert isinstance(res, float)
assert average_precision_score(np_y, np_y_pred) == pytest.approx(res)
def get_test_cases():
test_cases = [
# Binary input data of shape (N,) or (N, 1)
(torch.randint(0, 2, size=(50, )).long(),
torch.randint(0, 2, size=(50, )).long(), 1),
(torch.randint(0, 2, size=(50, 1)).long(),
torch.randint(0, 2, size=(50, 1)).long(), 1),
# updated batches
(torch.randint(0, 2, size=(50, )).long(),
torch.randint(0, 2, size=(50, )).long(), 16),
(torch.randint(0, 2, size=(50, 1)).long(),
torch.randint(0, 2, size=(50, 1)).long(), 16),
# Binary input data of shape (N, L)
(torch.randint(0, 2, size=(50, 4)).long(),
torch.randint(0, 2, size=(50, 4)).long(), 1),
(torch.randint(0, 2, size=(50, 7)).long(),
torch.randint(0, 2, size=(50, 7)).long(), 1),
# updated batches
(torch.randint(0, 2, size=(50, 4)).long(),
torch.randint(0, 2, size=(50, 4)).long(), 16),
(torch.randint(0, 2, size=(50, 7)).long(),
torch.randint(0, 2, size=(50, 7)).long(), 16),
]
return test_cases
for _ in range(5):
# check multiple random inputs as random exact occurencies are rare
test_cases = get_test_cases()
for y_pred, y, batch_size in test_cases:
_test(y_pred, y, batch_size)
def _test(n_epochs, metric_device):
metric_device = torch.device(metric_device)
n_iters = 80
s = 16
n_classes = 2
offset = n_iters * s
y_true = torch.randint(0,
n_classes,
size=(offset * idist.get_world_size(),
10)).to(device)
y_preds = torch.randint(0,
n_classes,
size=(offset * idist.get_world_size(),
10)).to(device)
def update(engine, i):
return (
y_preds[i * s + rank * offset:(i + 1) * s + rank * offset, :],
y_true[i * s + rank * offset:(i + 1) * s + rank * offset, :],
)
engine = Engine(update)
ap = AveragePrecision(device=metric_device)
ap.attach(engine, "ap")
data = list(range(n_iters))
engine.run(data=data, max_epochs=n_epochs)
assert "ap" in engine.state.metrics
res = engine.state.metrics["ap"]
if isinstance(res, torch.Tensor):
res = res.cpu().numpy()
true_res = average_precision_score(y_true.cpu().numpy(),
y_preds.cpu().numpy())
assert pytest.approx(res) == true_res
def _test(y_pred, y, batch_size):
def update_fn(engine, batch):
idx = (engine.state.iteration - 1) * batch_size
y_true_batch = np_y[idx:idx + batch_size]
y_pred_batch = np_y_pred[idx:idx + batch_size]
return idx, torch.from_numpy(y_pred_batch), torch.from_numpy(
y_true_batch)
engine = Engine(update_fn)
ap_metric = AveragePrecision(output_transform=lambda x: (x[1], x[2]))
ap_metric.attach(engine, "ap")
np_y = y.numpy()
np_y_pred = y_pred.numpy()
np_ap = average_precision_score(np_y, np_y_pred)
data = list(range(y_pred.shape[0] // batch_size))
ap = engine.run(data, max_epochs=1).metrics["ap"]
assert isinstance(ap, float)
assert np_ap == pytest.approx(ap)
def test_binary_input_N():
ap = AveragePrecision()
def _test(y_pred, y, n_iters):
ap.reset()
ap.update((y_pred, y))
np_y = y.numpy()
np_y_pred = y_pred.numpy()
if n_iters > 1:
batch_size = y.shape[0] // n_iters + 1
for i in range(n_iters):
idx = i * batch_size
ap.update(
(y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
res = ap.compute()
assert isinstance(res, float)
assert average_precision_score(np_y, np_y_pred) == pytest.approx(res)
def get_test_cases():
test_cases = [
(torch.randint(0, 2, size=(10, )).long(),
torch.randint(0, 2, size=(10, )).long(), 1),
(torch.randint(0, 2, size=(100, )).long(),
torch.randint(0, 2, size=(100, )).long(), 1),
(torch.randint(0, 2, size=(10, 1)).long(),
torch.randint(0, 2, size=(10, 1)).long(), 1),
(torch.randint(0, 2, size=(100, 1)).long(),
torch.randint(0, 2, size=(100, 1)).long(), 1),
# updated batches
(torch.randint(0, 2, size=(10, )).long(),
torch.randint(0, 2, size=(10, )).long(), 16),
(torch.randint(0, 2, size=(100, )).long(),
torch.randint(0, 2, size=(100, )).long(), 16),
(torch.randint(0, 2, size=(10, 1)).long(),
torch.randint(0, 2, size=(10, 1)).long(), 16),
(torch.randint(0, 2, size=(100, 1)).long(),
torch.randint(0, 2, size=(100, 1)).long(), 16),
]
return test_cases
for _ in range(10):
# check multiple random inputs as random exact occurencies are rare
test_cases = get_test_cases()
for y_pred, y, n_iters in test_cases:
_test(y_pred, y, n_iters)
def test_ap_score():
size = 100
np_y_pred = np.random.rand(size, 5)
np_y = np.random.randint(0, 2, size=(size, 5), dtype=np.long)
np_ap = average_precision_score(np_y, np_y_pred)
ap_metric = AveragePrecision()
y_pred = torch.from_numpy(np_y_pred)
y = torch.from_numpy(np_y)
ap_metric.reset()
ap_metric.update((y_pred, y))
ap = ap_metric.compute()
assert ap == np_ap
def test_ap_score_with_activation():
size = 100
np_y_pred = np.random.rand(size, 5)
np_y_pred_softmax = torch.softmax(torch.from_numpy(np_y_pred),
dim=1).numpy()
np_y = np.random.randint(0, 2, size=(size, 5), dtype=np.long)
np_ap = average_precision_score(np_y, np_y_pred_softmax)
ap_metric = AveragePrecision(activation=torch.nn.Softmax(dim=1))
y_pred = torch.from_numpy(np_y_pred)
y = torch.from_numpy(np_y)
ap_metric.reset()
ap_metric.update((y_pred, y))
ap = ap_metric.compute()
assert ap == np_ap
def run():
### Create tensorboardx writer
writer = create_summary_writer(model, train_loader, log_dir)
print("Writer created")
### Create trainer, evaulator
trainer = Engine(train_batch)
evaluator = Engine(eval_fn)
#### Attach evaluation metrics
Accuracy(output_transform=thresholded_output_transform).attach(
evaluator, 'accuracy')
Precision(output_transform=thresholded_output_transform).attach(
evaluator, 'precision')
Recall(output_transform=thresholded_output_transform).attach(
evaluator, 'recall')
AveragePrecision().attach(evaluator, 'AP')
Loss(criterion).attach(evaluator, 'loss')
#### tqdm settings
if USE_TQDM:
desc = "Epoch[{}] Iteration[{}/{}] Loss: {:.2f}"
pbar = tqdm(initial=0,
leave=False,
total=len(train_loader),
desc=desc.format(0, 0, 0, 0))
### Attach on events
@trainer.on(Events.ITERATION_COMPLETED)
def log_training_loss(engine):
iter = (engine.state.iteration - 1) % len(train_loader) + 1
if iter % log_interval == 0:
if USE_TQDM:
pbar.desc = desc.format(engine.state.epoch, iter,
len(train_loader), engine.state.output)
pbar.update(log_interval)
else:
print("Epoch[{}] Iteration[{}/{}] Loss: {:.2f}".format(
engine.state.epoch, iter, len(train_loader),
engine.state.output))
writer.add_scalar("training/loss", engine.state.output,
engine.state.iteration)
@trainer.on(Events.EPOCH_COMPLETED)
def log_training_results(engine):
evaluator.run(train_loader)
metrics = evaluator.state.metrics
avg_accuracy = metrics['accuracy']
avg_loss = metrics['loss']
prec = metrics['precision']
recall = metrics['recall']
ap = metrics['AP']
print(
"Training Results - Epoch: {} Avg accuracy: {:.2f} Avg loss: {:.2f} Precision: {:.2f} Recall: {:.2f} APR: {:.2f}"
.format(engine.state.epoch, avg_accuracy, avg_loss, prec, recall,
ap))
writer.add_scalar("training/avg_loss", avg_loss, engine.state.epoch)
writer.add_scalar("training/avg_accuracy", avg_accuracy,
engine.state.epoch)
writer.add_scalar("training/precision", prec, engine.state.epoch)
writer.add_scalar("training/recall", recall, engine.state.epoch)
writer.add_scalar("training/avg precision", ap, engine.state.epoch)
@trainer.on(Events.EPOCH_COMPLETED)
def log_val_results(engine):
evaluator.run(val_loader)
metrics = evaluator.state.metrics
avg_accuracy = metrics['accuracy']
avg_loss = metrics['loss']
prec = metrics['precision']
recall = metrics['recall']
ap = metrics['AP']
print(
"Training Results - Epoch: {} Avg accuracy: {:.2f} Avg loss: {:.2f} Precision: {:.2f} Recall: {:.2f} APR: {:.2f}"
.format(engine.state.epoch, avg_accuracy, avg_loss, prec, recall,
ap))
writer.add_scalar("validation/avg_loss", avg_loss, engine.state.epoch)
writer.add_scalar("validation/avg_accuracy", avg_accuracy,
engine.state.epoch)
writer.add_scalar("v[{}] Iteration[{}/{}] Loss:alidation/precision",
prec, engine.state.epoch)
writer.add_scalar("validation/recall", recall, engine.state.epoch)
writer.add_scalar("validation/avg precision", ap, engine.state.epoch)
#### Run the joint
print("Training...")
trainer.run(train_loader, max_epochs=epochs)
writer.close()
def train(epochs: int, model: nn.Module, train_loader: DataLoader,
valid_loader: DataLoader, criterion: Callable, device: str,
lr: float, patience: int, lr_decay: float, lr_scheduler: str,
lr_scheduler_kwargs: Dict[str, Any]):
model.to(torch.device(device))
optimizer = optim.Adam(
[param for param in model.parameters() if param.requires_grad], lr=lr)
trainer = create_supervised_trainer(model,
optimizer,
criterion,
device=device)
scheduler = LRScheduler(
getattr(optim.lr_scheduler, lr_scheduler)(optimizer,
**lr_scheduler_kwargs))
trainer.add_event_handler(Events.ITERATION_COMPLETED, scheduler)
pbar = ProgressBar(False)
pbar.attach(trainer)
train_evaluator = create_supervised_evaluator(
model,
metrics={
'ACC': Accuracy(discreted_output_transform),
'BCE': Loss(criterion),
'AP': AveragePrecision(probability_output_transform)
},
device=device)
valid_evaluator = create_supervised_evaluator(
model,
metrics={
'ACC': Accuracy(discreted_output_transform),
'BCE': Loss(criterion),
'AP': AveragePrecision(probability_output_transform)
},
device=device)
history = {
col: list()
for col in [
'epoch', 'elapsed time', 'iterations', 'lr', 'train BCE',
'valid BCE', 'train ACC', 'valid ACC', 'train AP', 'valid AP'
]
}
@trainer.on(Events.EPOCH_COMPLETED)
def log_training_results(engine):
train_evaluator.run(train_loader)
history['train BCE'] += [train_evaluator.state.metrics['BCE']]
history['train ACC'] += [train_evaluator.state.metrics['ACC']]
history['train AP'] += [train_evaluator.state.metrics['AP']]
@trainer.on(Events.EPOCH_COMPLETED)
def log_validation_results(engine):
valid_evaluator.run(valid_loader)
history['epoch'] += [valid_evaluator.state.epoch]
history['iterations'] += [valid_evaluator.state.epoch_length]
history['elapsed time'] += [
0 if len(history['elapsed time']) == 0 else
history['elapsed time'][-1] +
valid_evaluator.state.times['COMPLETED']
]
history['lr'] += [scheduler.get_param()]
history['valid BCE'] += [valid_evaluator.state.metrics['BCE']]
history['valid ACC'] += [valid_evaluator.state.metrics['ACC']]
history['valid AP'] += [valid_evaluator.state.metrics['AP']]
@trainer.on(Events.EPOCH_COMPLETED)
def log_progress_bar(engine):
pbar.log_message(
f"train BCE: {history['train BCE'][-1]:.2f} " \
+ f"train ACC: {history['train ACC'][-1]:.2f} " \
+ f"train AP: {history['train AP'][-1]:.2f} " \
+ f"valid BCE: {history['valid BCE'][-1]:.2f} " \
+ f"valid ACC: {history['valid ACC'][-1]:.2f} " \
+ f"valid AP: {history['valid AP'][-1]:.2f}"
)
# Early stopping
handler = EarlyStopping(patience=patience,
score_function=score_function,
trainer=trainer)
valid_evaluator.add_event_handler(Events.EPOCH_COMPLETED, handler)
trainer.run(train_loader, max_epochs=epochs)
return pd.DataFrame(history)
def test_input_types():
ap = AveragePrecision()
ap.reset()
output1 = (torch.rand(4,
3), torch.randint(0,
2,
size=(4, 3),
dtype=torch.long))
ap.update(output1)
with pytest.raises(
ValueError,
match=
r"Incoherent types between input y_pred and stored predictions"):
ap.update((torch.randint(0, 5,
size=(4, 3)), torch.randint(0, 2,
size=(4, 3))))
with pytest.raises(
ValueError,
match=r"Incoherent types between input y and stored targets"):
ap.update((torch.rand(4,
3), torch.randint(0, 2,
size=(4, 3)).to(torch.int32)))
with pytest.raises(
ValueError,
match=
r"Incoherent types between input y_pred and stored predictions"):
ap.update((torch.randint(0, 2, size=(10, )).long(),
torch.randint(0, 2, size=(10, 5)).long()))
def test_no_sklearn(mock_no_sklearn):
with pytest.raises(
RuntimeError,
match=r"This contrib module requires sklearn to be installed."):
AveragePrecision()
def train(config):
model_suite.logging.setup_loggers(config)
device = 'cpu'
if torch.cuda.is_available():
device = 'cuda'
logger.info(f'Device {device} will be used')
data_df = read_data()
train_ds, val_ds, test_ds = get_datasets(data_df)
train_loader, val_loader = get_data_loaders(
train_ds,
val_ds,
train_batch_size=config.train_batch_size,
val_batch_size=config.val_batch_size)
writer = SummaryWriter(log_dir=f'{config.model_dir}/logs')
n_features = train_loader.dataset[0][0].shape[0]
model = get_model(model_name=config.model, n_features=n_features)
loss = torch.nn.BCEWithLogitsLoss()
optimizer = Adam(model.parameters(),
lr=config.learning_rate,
weight_decay=config.weight_decay)
#optimizer = SGD(model.parameters(), lr=config.learning_rate, weight_decay=config.weight_decay, momentum=config.momentum)
trainer = create_supervised_trainer(model, optimizer, loss, device=device)
evaluator = create_supervised_evaluator(model,
metrics={
'loss': Loss(loss),
'roc': ROC_AUC(),
'accuracy': Accuracy(),
'precision':
AveragePrecision()
},
device=device)
@trainer.on(Events.EPOCH_COMPLETED)
def log_training_results(engine):
evaluator.run(train_loader)
metrics = evaluator.state.metrics
avg_loss = metrics['loss']
avg_roc = metrics['roc']
# avg_accuracy = metrics['accuracy']
# avg_precision = metrics['precision']
logger.info(
f'Training results - Epoch: {engine.state.epoch} Avg loss: {avg_loss} ROC: {avg_roc}'
)
writer.add_scalar("training/avg_loss", avg_loss, engine.state.epoch)
writer.add_scalar("training/avg_roc", avg_roc, engine.state.epoch)
# writer.add_scalar("training/avg_accuracy", avg_accuracy, engine.state.epoch)
# writer.add_scalar("training/avg_precision", avg_precision, engine.state.epoch)
@trainer.on(Events.EPOCH_COMPLETED)
def log_validation_results(engine):
evaluator.run(val_loader)
metrics = evaluator.state.metrics
avg_loss = metrics['loss']
avg_roc = metrics['roc']
# avg_accuracy = metrics['accuracy']
# avg_precision = metrics['precision']
logger.info(
f'Validation results - Epoch: {engine.state.epoch} Avg loss: {avg_loss} ROC: {avg_roc}'
)
writer.add_scalar("valdation/avg_loss", avg_loss, engine.state.epoch)
writer.add_scalar("valdation/avg_roc", avg_roc, engine.state.epoch)
# writer.add_scalar("valdation/avg_accuracy", avg_accuracy, engine.state.epoch)
# writer.add_scalar("valdation/avg_precision", avg_precision, engine.state.epoch)
trainer.run(train_loader, max_epochs=config.n_epochs)
writer.close()
cycle_size=len(train_loader) * cfg.n_epochs,
start_value_mult=0,
end_value_mult=0),
warmup_start_value=0.0,
warmup_end_value=cfg.lr,
warmup_duration=len(train_loader)))
evaluator = create_supervised_evaluator(
model,
metrics={
'loss': Loss(loss),
'acc_smpl': Accuracy(threshold_output, is_multilabel=True),
'p': Precision(threshold_output, average=True),
'r': Recall(threshold_output, average=True),
'f1': Fbeta(1.0, output_transform=threshold_output),
'ap': AveragePrecision(output_transform=activate_output)
},
device=device)
model_checkpoint = ModelCheckpoint(
dirname=wandb.run.dir,
filename_prefix='best',
require_empty=False,
score_function=lambda e: e.state.metrics['ap'],
global_step_transform=global_step_from_engine(trainer))
evaluator.add_event_handler(Events.COMPLETED, model_checkpoint,
{'model': model})
@trainer.on(Events.EPOCH_COMPLETED)
def validate(trainer):
evaluator.run(val_loader)
