def test_binary_wrong_inputs():
re = Recall()
assert re._updated is False
with pytest.raises(
ValueError,
match=r"For binary cases, y must be comprised of 0's and 1's"):
# y has not only 0 or 1 values
re.update((torch.randint(0, 2, size=(10, )), torch.arange(0,
10).long()))
assert re._updated is False
with pytest.raises(
ValueError,
match=r"For binary cases, y_pred must be comprised of 0's and 1's"
):
# y_pred values are not thresholded to 0, 1 values
re.update((torch.rand(10, 1), torch.randint(0, 2, size=(10, )).long()))
assert re._updated is False
with pytest.raises(ValueError, match=r"y must have shape of"):
# incompatible shapes
re.update((torch.randint(0, 2, size=(10, )),
torch.randint(0, 2, size=(10, 5)).long()))
assert re._updated is False
with pytest.raises(ValueError, match=r"y must have shape of"):
# incompatible shapes
re.update((torch.randint(0, 2, size=(10, 5, 6)),
torch.randint(0, 2, size=(10, )).long()))
assert re._updated is False
with pytest.raises(ValueError, match=r"y must have shape of"):
# incompatible shapes
re.update((torch.randint(0, 2, size=(10, )),
torch.randint(0, 2, size=(10, 5, 6)).long()))
assert re._updated is False
'optimizer': Adam(gru.parameters(), lr=learning_rate),
'loss': nn.CrossEntropyLoss(),
'epoch': 5,
},
{
'nn': bgru,
'optimizer': Adam(bgru.parameters(), lr=learning_rate),
'loss': nn.CrossEntropyLoss(),
'epoch': 5,
},
]
# set metrics
metrics = {
'prec': Precision(average=True),
'recall': Recall(average=True),
}
# user-defined event handler
def log_iter_10(trainer, evaluator, train, valid):
iter_num = trainer.state.iteration
epoch_num = trainer.state.epoch
loss = trainer.state.output
if iter_num % 10 == 0:
logger.info(f"Epoch[{epoch_num}] Iter: {iter_num} Loss: {loss:.2}")
def main():
# logger config
logger.level = 'debug'
def test_binary_wrong_inputs():
re = Recall()
with pytest.raises(ValueError):
# y has not only 0 or 1 values
re.update((torch.randint(0, 2, size=(10, )),
torch.arange(0, 10).type(torch.LongTensor)))
with pytest.raises(ValueError):
# y_pred values are not thresholded to 0, 1 values
re.update(
(torch.rand(10,
1), torch.randint(0, 2,
size=(10, )).type(torch.LongTensor)))
with pytest.raises(ValueError):
# incompatible shapes
re.update((torch.randint(0, 2, size=(10, )),
torch.randint(0, 2, size=(10, 5)).type(torch.LongTensor)))
with pytest.raises(ValueError):
# incompatible shapes
re.update((torch.randint(0, 2, size=(10, 5, 6)),
torch.randint(0, 2, size=(10, )).type(torch.LongTensor)))
with pytest.raises(ValueError):
# incompatible shapes
re.update((torch.randint(0, 2, size=(10, )),
torch.randint(0, 2,
size=(10, 5, 6)).type(torch.LongTensor)))
def test_multiclass_wrong_inputs():
re = Recall()
with pytest.raises(ValueError):
# incompatible shapes
re.update((torch.rand(10, 5, 4), torch.randint(0, 2,
size=(10, )).long()))
with pytest.raises(ValueError):
# incompatible shapes
re.update((torch.rand(10, 5, 6), torch.randint(0, 5,
size=(10, 5)).long()))
with pytest.raises(ValueError):
# incompatible shapes
re.update((torch.rand(10), torch.randint(0, 5,
size=(10, 5, 6)).long()))
re = Recall(average=True)
with pytest.raises(ValueError):
# incompatible shapes between two updates
re.update((torch.rand(10, 5), torch.randint(0, 5, size=(10, )).long()))
re.update((torch.rand(10, 6), torch.randint(0, 5, size=(10, )).long()))
with pytest.raises(ValueError):
# incompatible shapes between two updates
re.update((torch.rand(10, 5, 12,
14), torch.randint(0, 5,
size=(10, 12, 14)).long()))
re.update((torch.rand(10, 6, 12,
14), torch.randint(0, 5,
size=(10, 12, 14)).long()))
re = Recall(average=False)
with pytest.raises(ValueError):
# incompatible shapes between two updates
re.update((torch.rand(10, 5), torch.randint(0, 5, size=(10, )).long()))
re.update((torch.rand(10, 6), torch.randint(0, 5, size=(10, )).long()))
with pytest.raises(ValueError):
# incompatible shapes between two updates
re.update((torch.rand(10, 5, 12,
14), torch.randint(0, 5,
size=(10, 12, 14)).long()))
re.update((torch.rand(10, 6, 12,
14), torch.randint(0, 5,
size=(10, 12, 14)).long()))
def _test(average):
re = Recall(average=average, is_multilabel=True)
y_pred = torch.randint(0, 2, size=(10, 5, 18, 16))
y = torch.randint(0, 2, size=(10, 5, 18, 16)).long()
re.update((y_pred, y))
np_y_pred = to_numpy_multilabel(y_pred)
np_y = to_numpy_multilabel(y)
assert re._type == "multilabel"
re_compute = re.compute() if average else re.compute().mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert recall_score(np_y, np_y_pred,
average="samples") == pytest.approx(re_compute)
re.reset()
y_pred = torch.randint(0, 2, size=(10, 4, 20, 23))
y = torch.randint(0, 2, size=(10, 4, 20, 23)).long()
re.update((y_pred, y))
np_y_pred = to_numpy_multilabel(y_pred)
np_y = to_numpy_multilabel(y)
assert re._type == "multilabel"
re_compute = re.compute() if average else re.compute().mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert recall_score(np_y, np_y_pred,
average="samples") == pytest.approx(re_compute)
# Batched Updates
re.reset()
y_pred = torch.randint(0, 2, size=(100, 5, 12, 14))
y = torch.randint(0, 2, size=(100, 5, 12, 14)).long()
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
re.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
np_y = to_numpy_multilabel(y)
np_y_pred = to_numpy_multilabel(y_pred)
assert re._type == "multilabel"
re_compute = re.compute() if average else re.compute().mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert recall_score(np_y, np_y_pred,
average="samples") == pytest.approx(re_compute)
def run(tb, vb, lr, epochs, writer):
device = os.environ['main-device']
logging.info('Training program start!')
logging.info('Configuration:')
logging.info('\n'+json.dumps(INFO, indent=2))
# ------------------------------------
# 1. Define dataloader
train_loader, train4val_loader, val_loader, num_of_images, mapping = get_dataloaders(tb, vb)
# train_loader, train4val_loader, val_loader, num_of_images = get_dataloaders(tb, vb)
weights = (1/num_of_images)/((1/num_of_images).sum().item())
# weights = (1/num_of_images)/(1/num_of_images + 1/(num_of_images.sum().item()-num_of_images))
weights = weights.to(device=device)
# ------------------------------------
# 2. Define model
model = EfficientNet.from_pretrained('efficientnet-b3', num_classes=INFO['dataset-info']['num-of-classes'])
model = carrier(model)
# ------------------------------------
# 3. Define optimizer
optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200)
ignite_scheduler = LRScheduler(scheduler)
# ------------------------------------
# 4. Define metrics
# class SoftCrossEntropyLoss(nn.Module):
# def __init__(self, weight=None):
# super(SoftCrossEntropyLoss, self).__init__()
# self.class_weights = weight
# def forward(self, input, target):
# softmax = torch.exp(input) / torch.exp(input).sum(1)[:, None]
# onehot_labels = to_onehot(target, input.shape[1])
# soft_labels = torch.zeros_like(onehot_labels)
# soft_labels = torch.where(onehot_labels.cpu() == 1, torch.tensor([0.9]), torch.tensor([0.1/(input.shape[1]-1)])).to(device=device)
# if self.class_weights is not None:
# # print(soft_labels.shape, softmax.shape)
# loss = -torch.sum(torch.log(softmax) * soft_labels * self.class_weights * input.shape[1])
# else:
# loss = -torch.sum(torch.log(softmax) * soft_labels)
# return loss
class LabelMSELoss(nn.Module):
def __init__(self, weight=None):
super(LabelMSELoss, self).__init__()
self.class_weights = weight.to(device=device)
def forward(self, input, target):
target_onehot = to_onehot(target, num_classes=input.shape[1]).to(device=device)
mse = (input - target_onehot) ** 2
if self.class_weights is not None:
weights = self.class_weights[target] * input.shape[1]
return (mse.sum(1) * weights).sum()
else:
return mse.sum()
class EntropyPrediction(metric.Metric):
def __init__(self, threshold=0.5):
super(EntropyPrediction, self).__init__()
self.threshold = threshold
self.prediction = torch.tensor([], dtype=torch.int)
self.y = torch.tensor([], dtype=torch.int)
def reset(self):
# self.threshold = 0.5
self.prediction = torch.tensor([])
self.y = torch.tensor([])
super(EntropyPrediction, self).reset()
def update(self, output):
y_pred, y = output
softmax = torch.exp(y_pred) / torch.exp(y_pred).sum(1)[:, None]
entropy_base = math.log(y_pred.shape[1])
entropy = (-softmax * torch.log(softmax)).sum(1)/entropy_base
values, inds = softmax.max(1)
prediction = torch.where(entropy<self.threshold, inds, torch.tensor([-1]).to(device=device))
self.prediction = torch.cat((self.prediction.type(torch.LongTensor).to(device=device), torch.tensor([mapping[x.item()] for x in prediction]).to(device=device)))
self.y = torch.cat((self.y.type(torch.LongTensor).to(device=device), y.to(device=device)))
# return self.prediction, self.y
def compute(self):
return self.prediction, self.y
train_metrics = {
'accuracy': Accuracy(),
'loss': Loss(LabelMSELoss()),
'precision_recall': MetricsLambda(PrecisionRecallTable, Precision(), Recall(), train_loader.dataset.classes),
'cmatrix': MetricsLambda(CMatrixTable, ConfusionMatrix(INFO['dataset-info']['num-of-classes']), train_loader.dataset.classes)
}
val_metrics = {
'accuracy': MetricsLambda(Labels2Acc, EntropyPrediction(1.0)),
'precision_recall': MetricsLambda(Labels2PrecisionRecall, EntropyPrediction(1.0), val_loader.dataset.classes),
'cmatrix': MetricsLambda(Labels2CMatrix, EntropyPrediction(1.0), val_loader.dataset.classes)
}
# ------------------------------------
# 5. Create trainer
trainer = create_supervised_trainer(model, optimizer, LabelMSELoss(), device=device)
# ------------------------------------
# 6. Create evaluator
train_evaluator = create_supervised_evaluator(model, metrics=train_metrics, device=device)
val_evaluator = create_supervised_evaluator(model, metrics=val_metrics, device=device)
desc = 'ITERATION - loss: {:.4f}'
pbar = tqdm(
initial=0, leave=False, total=len(train_loader),
desc=desc.format(0)
)
# ------------------------------------
# 7. Create event hooks
# Update process bar on each iteration completed.
@trainer.on(Events.ITERATION_COMPLETED)
def log_training_loss(engine):
log_interval = 1
iter = (engine.state.iteration - 1) % len(train_loader) + 1
if iter % log_interval == 0:
pbar.desc = desc.format(engine.state.output)
pbar.update(log_interval)
@trainer.on(Events.EPOCH_STARTED)
def refresh_pbar(engine):
pbar.refresh()
pbar.n = pbar.last_print_n = 0
# Compute metrics on train data on each epoch completed.
@trainer.on(Events.EPOCH_COMPLETED)
def log_training_results(engine):
pbar.refresh()
print ('Checking on training set.')
train_evaluator.run(train4val_loader)
metrics = train_evaluator.state.metrics
avg_accuracy = metrics['accuracy']
avg_loss = metrics['loss']
precision_recall = metrics['precision_recall']
cmatrix = metrics['cmatrix']
prompt = """
Training Results - Epoch: {}
Avg accuracy: {:.4f}
Avg loss: {:.4f}
precision_recall: \n{}
confusion matrix: \n{}
""".format(engine.state.epoch,avg_accuracy,avg_loss,precision_recall['pretty'],cmatrix['pretty'])
tqdm.write(prompt)
logging.info('\n'+prompt)
writer.add_text(os.environ['run-id'], prompt, engine.state.epoch)
writer.add_scalars('Aggregate/Acc', {'Train Acc': avg_accuracy}, engine.state.epoch)
writer.add_scalars('Aggregate/Loss', {'Train Loss': avg_loss}, engine.state.epoch)
# Compute metrics on val data on each epoch completed.
@trainer.on(Events.EPOCH_COMPLETED)
def log_validation_results(engine):
pbar.clear()
print('* - * - * - * - * - * - * - * - * - * - * - * - *')
print ('Checking on validation set.')
val_evaluator.run(val_loader)
metrics = val_evaluator.state.metrics
avg_accuracy = metrics['accuracy']
precision_recall = metrics['precision_recall']
cmatrix = metrics['cmatrix']
prompt = """
Validating Results - Epoch: {}
Avg accuracy: {:.4f}
precision_recall: \n{}
confusion matrix: \n{}
""".format(engine.state.epoch,avg_accuracy,precision_recall['pretty'],cmatrix['pretty'])
tqdm.write(prompt)
logging.info('\n'+prompt)
writer.add_text(os.environ['run-id'], prompt, engine.state.epoch)
writer.add_scalars('Aggregate/Acc', {'Val Acc': avg_accuracy}, engine.state.epoch)
writer.add_scalars('Aggregate/Score', {'Val avg precision': precision_recall['data'][0, -1], 'Val avg recall': precision_recall['data'][1, -1]}, engine.state.epoch)
# Save model ever N epoch.
save_model_handler = ModelCheckpoint(os.environ['savedir'], '', save_interval=10, n_saved=2)
trainer.add_event_handler(Events.EPOCH_COMPLETED, save_model_handler, {'model': model})
# Update learning-rate due to scheduler.
trainer.add_event_handler(Events.EPOCH_STARTED, ignite_scheduler)
# ------------------------------------
# Run
trainer.run(train_loader, max_epochs=epochs)
pbar.close()
def test_no_update():
recall = Recall()
with pytest.raises(NotComputableError):
recall.compute()
batch=batch,
train_step=train_step,
device="cpu")
print("SUCCESS: variables changed")
except Exception as e:
print("FAILED: ", e)
exit(1)
test_variables_change(model, model_data[0])
trainer = Engine(update_model)
evaluator = Engine(predict_on_batch)
Accuracy().attach(evaluator, "accuracy")
Precision().attach(evaluator, "precision")
Recall().attach(evaluator, "recall")
@trainer.on(Events.ITERATION_COMPLETED(every=100))
def log_training(engine):
batch_loss = engine.state.output
lr = optimizer.param_groups[0]['lr']
epoch = engine.state.epoch
max_epochs = engine.state.max_epochs
iteration = engine.state.iteration
print(
f"Epoch {epoch}/{max_epochs} : {iteration} - batch loss: {batch_loss: .4f}, lr: {lr}"
)
@trainer.on(Events.EPOCH_COMPLETED)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
home = os.environ['HOME']
writer = SummaryWriter(home + '/log.json')
lowest_loss = np.Inf
train_loader, val_loader, test_loader = load_data(args.batch_size)
model.to(device)
if args.summary:
summary(model, (1, 28, 28))
print('Batch size: ', args.batch_size)
print('Epochs:', args.num_epochs)
trainer = create_supervised_trainer(model, adam_optimizer, cross_entropy_loss, device=device)
evaluator = create_supervised_evaluator(model, metrics={"accuracy": Accuracy(), "cross": Loss(cross_entropy_loss),
"prec": Precision(), "recall": Recall()},
device=device)
desc = "ITERATION - loss: {:.2f}"
pbar = tqdm(
initial=0, leave=False, total=len(train_loader),
desc=desc.format(0)
)
@trainer.on(Events.ITERATION_COMPLETED)
def log_training_loss(engine):
iter = (engine.state.iteration - 1) % len(train_loader) + 1
if iter % 10 == 0:
pbar.desc = desc.format(engine.state.output)
def create_supervised_evaluator(
model: torch.nn.Module,
prepare_batch,
criterion,
metrics=None,
device=None,
non_blocking: bool = False,
tqdm_log: bool = False,
checkpoint_dir='output/checkpoints/'
) -> Engine:
if device:
model.to(device)
def _inference(engine, batch):
model.eval()
with torch.no_grad():
actions, target = prepare_batch(
batch, device=device, non_blocking=non_blocking)
scores = model(actions)
return (scores, target)
engine = Engine(_inference)
softmax_transform = lambda x:\
(F.softmax(x[0], dim=1)[:, 1] > 0.5, x[1])
Loss(
criterion, output_transform=lambda x: x,
).attach(engine, 'loss')
ROC_AUC(
output_transform=lambda x: (F.softmax(x[0], dim=1)[:, 1], x[1])
).attach(engine, 'roc_auc')
ModdedPrecision(
output_transform=softmax_transform
).attach(engine, 'precision')
Recall(
output_transform=softmax_transform
).attach(engine, 'recall')
FalsePositiveRate(
output_transform=softmax_transform
).attach(engine, 'FPR')
if tqdm_log:
pbar = ProgressBar(persist=True)
pbar.attach(engine)
# save the best model
# to_save = {'model': model}
# best_checkpoint_handler = Checkpoint(
# to_save,
# DiskSaver(checkpoint_dir, create_dir=True),
# n_saved=1,
# filename_prefix='best',
# score_function=lambda x: engine.state.metrics['roc_auc'],
# score_name="roc_auc",
# global_step_transform=lambda x, y : engine.train_epoch)
# engine.add_event_handler(Events.COMPLETED, best_checkpoint_handler)
@engine.on(Events.COMPLETED)
def log_validation_results(engine):
metrics = engine.state.metrics
if len(metrics) == 0:
print('no metrics in log_validation_results!')
return
print(f"{'Validation Results':20} - "
f"Avg loss: {metrics['loss']:.6f}, "
f"ROC AUC: {metrics['roc_auc']:.6f}\n\t"
f"Recall: {metrics['recall']:.6f} "
f"Precision: {metrics['precision']:.6f} "
f"FPR: {metrics['FPR']:.6f} "
)
wandb.log({
"val_loss": metrics['loss'],
"val_roc_auc": metrics['roc_auc'],
"val_recall": metrics['recall'],
"val_precision": metrics['precision'],
"val_fpr": metrics['FPR']
}, commit=True)
return engine
def create_supervised_trainer(model,
optimizer,
criterion,
prepare_batch,
metrics={},
device=None,
tqdm_log=False,
) -> Engine:
def _update(engine, batch):
model.train()
optimizer.zero_grad()
actions, target = prepare_batch(batch, device=device)
scores = model(actions)
loss = criterion(scores, target)
loss.backward()
optimizer.step()
return {'loss': loss.item(), 'y_pred': scores, 'y_true': target}
model.to(device)
engine = Engine(_update)
softmax_transform = lambda x:\
(F.softmax(x['y_pred'], dim=1)[:, 1] > 0.5, x['y_true'])
# Metrics
RunningAverage(output_transform=lambda x: x['loss'])\
.attach(engine, 'running_average_loss')
Loss(
criterion, output_transform=lambda x: (x['y_pred'], x['y_true']),
).attach(engine, 'loss')
ROC_AUC(
output_transform=lambda x: (F.softmax(x['y_pred'], dim=1)[:, 1], x['y_true'])
).attach(engine, 'roc_auc')
ModdedPrecision(
output_transform=softmax_transform
).attach(engine, 'precision')
Recall(
output_transform=softmax_transform
).attach(engine, 'recall')
FalsePositiveRate(
output_transform=softmax_transform
).attach(engine, 'FPR')
# TQDM
if tqdm_log:
pbar = ProgressBar(
persist=True,
)
pbar.attach(engine, ['average_loss'])
@engine.on(Events.EPOCH_COMPLETED)
def log_validation_results(engine):
metrics = engine.state.metrics
print(f"Epoch {engine.state.epoch} completed!")
print(f"{'Train Results':20} - "
f"Avg loss: {metrics['loss']:.6f}, "
f"ROC AUC: {metrics['roc_auc']:.6f}\n\t"
f"Recall: {metrics['recall']:.6f} "
f"Precision: {metrics['precision']:.6f} "
f"FPR: {metrics['FPR']:.6f} "
)
wandb.log({
"train_loss": metrics['loss'],
"train_roc_auc": metrics['roc_auc'],
"train_recall": metrics['recall'],
"train_precision": metrics['precision'],
"train_fpr": metrics['FPR']
}, commit=False)
return engine
def train(name, load, lrate, weight_decay, workers, smooth, device, validation,
ground_truth):
if not name:
name = '{}_{}'.format(lrate, weight_decay)
click.echo('model output name: {}'.format(name))
torch.set_num_threads(1)
train_set = BaselineSet(glob.glob('{}/**/*.seeds.png'.format(ground_truth),
recursive=True),
smooth=smooth)
train_data_loader = DataLoader(dataset=train_set,
num_workers=workers,
batch_size=1,
shuffle=True,
pin_memory=True)
val_set = BaselineSet(glob.glob('{}/**/*.seeds.png'.format(validation),
recursive=True),
smooth=smooth)
val_data_loader = DataLoader(dataset=val_set,
num_workers=workers,
batch_size=1,
pin_memory=True)
click.echo('loading network')
model = ResUNet(refine_encoder=False).to(device)
if load:
click.echo('loading weights')
model = torch.load(load, map_location=device)
criterion = nn.BCEWithLogitsLoss()
opti = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=lrate,
weight_decay=weight_decay)
def score_function(engine):
val_loss = engine.state.metrics['loss']
return -val_loss
def output_preprocess(output):
o, target = output
o = torch.sigmoid(o)
o = denoising_hysteresis_thresh(o.detach().squeeze().cpu().numpy(),
0.8, 0.9, 2.5)
return torch.from_numpy(o.astype('f')).unsqueeze(0).unsqueeze(0).to(
device), target.double().to(device)
trainer = create_supervised_trainer(model,
opti,
criterion,
device=device,
non_blocking=True)
accuracy = Accuracy(output_transform=output_preprocess)
precision = Precision(output_transform=output_preprocess)
recall = Recall(output_transform=output_preprocess)
loss = Loss(criterion)
precision = Precision(average=False)
recall = Recall(average=False)
f1 = (precision * recall * 2 / (precision + recall)).mean()
evaluator = create_supervised_evaluator(model,
device=device,
non_blocking=True)
accuracy.attach(evaluator, 'accuracy')
precision.attach(evaluator, 'precision')
recall.attach(evaluator, 'recall')
loss.attach(evaluator, 'loss')
f1.attach(evaluator, 'f1')
ckpt_handler = ModelCheckpoint('.',
name,
save_interval=1,
n_saved=10,
require_empty=False)
RunningAverage(output_transform=lambda x: x).attach(trainer, 'loss')
progress_bar = ProgressBar(persist=True)
progress_bar.attach(trainer, ['loss'])
trainer.add_event_handler(event_name=Events.EPOCH_COMPLETED,
handler=ckpt_handler,
to_save={'net': model})
trainer.add_event_handler(event_name=Events.ITERATION_COMPLETED,
handler=TerminateOnNan())
@trainer.on(Events.EPOCH_COMPLETED)
def log_validation_results(engine):
evaluator.run(val_data_loader)
metrics = evaluator.state.metrics
progress_bar.log_message(
'eval results - epoch {} loss: {:.4f} f1: {:.4f}, accuracy: {:.4f} recall: {:.4f} precision {:.4f}'
.format(engine.state.epoch, metrics['loss'], metrics['f1'],
metrics['accuracy'], metrics['recall'],
metrics['precision']))
trainer.run(train_data_loader, max_epochs=1000)
def test_binary_input(average):
re = Recall(average=average)
assert re._updated is False
def _test(y_pred, y, batch_size):
re.reset()
assert re._updated is False
if batch_size > 1:
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
re.update(
(y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
else:
re.update((y_pred, y))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert re._type == "binary"
assert re._updated is True
assert isinstance(re.compute(), float if average else torch.Tensor)
re_compute = re.compute() if average else re.compute().numpy()
assert recall_score(np_y, np_y_pred,
average="binary") == pytest.approx(re_compute)
def get_test_cases():
test_cases = [
# Binary accuracy on input of shape (N, 1) or (N, )
(torch.randint(0, 2, size=(10, )), torch.randint(0, 2,
size=(10, )), 1),
(torch.randint(0, 2,
size=(10, 1)), torch.randint(0, 2,
size=(10, 1)), 1),
# updated batches
(torch.randint(0, 2, size=(50, )), torch.randint(0, 2,
size=(50, )), 16),
(torch.randint(0, 2,
size=(50, 1)), torch.randint(0, 2,
size=(50, 1)), 16),
# Binary accuracy on input of shape (N, L)
(torch.randint(0, 2,
size=(10, 5)), torch.randint(0, 2,
size=(10, 5)), 1),
(torch.randint(0, 2, size=(10, 1, 5)),
torch.randint(0, 2, size=(10, 1, 5)), 1),
# updated batches
(torch.randint(0, 2,
size=(50, 5)), torch.randint(0, 2,
size=(50, 5)), 16),
(torch.randint(0, 2, size=(50, 1, 5)),
torch.randint(0, 2, size=(50, 1, 5)), 16),
# Binary accuracy on input of shape (N, H, W)
(torch.randint(0, 2, size=(10, 12, 10)),
torch.randint(0, 2, size=(10, 12, 10)), 1),
(torch.randint(0, 2, size=(10, 1, 12, 10)),
torch.randint(0, 2, size=(10, 1, 12, 10)), 1),
# updated batches
(torch.randint(0, 2, size=(50, 12, 10)),
torch.randint(0, 2, size=(50, 12, 10)), 16),
(torch.randint(0, 2, size=(50, 1, 12, 10)),
torch.randint(0, 2, size=(50, 1, 12, 10)), 16),
# Corner case with all zeros predictions
(torch.zeros(size=(10, )), torch.randint(0, 2, size=(10, )), 1),
(torch.zeros(size=(10, 1)), torch.randint(0, 2, size=(10, 1)), 1),
]
return test_cases
for _ in range(5):
# check multiple random inputs as random exact occurencies are rare
test_cases = get_test_cases()
for y_pred, y, batch_size in test_cases:
_test(y_pred, y, batch_size)
def _test_distrib_integration_multilabel(device):
from ignite.engine import Engine
rank = idist.get_rank()
torch.manual_seed(12)
def _test(average, n_epochs, metric_device):
n_iters = 60
s = 16
n_classes = 7
offset = n_iters * s
y_true = torch.randint(0,
2,
size=(offset * idist.get_world_size(),
n_classes, 6, 8)).to(device)
y_preds = torch.randint(0,
2,
size=(offset * idist.get_world_size(),
n_classes, 6, 8)).to(device)
def update(engine, i):
return (
y_preds[i * s + rank * offset:(i + 1) * s + rank * offset,
...],
y_true[i * s + rank * offset:(i + 1) * s + rank * offset, ...],
)
engine = Engine(update)
re = Recall(average=average, is_multilabel=True, device=metric_device)
re.attach(engine, "re")
assert re._updated is False
data = list(range(n_iters))
engine.run(data=data, max_epochs=n_epochs)
assert "re" in engine.state.metrics
assert re._updated is True
res = engine.state.metrics["re"]
res2 = re.compute()
if isinstance(res, torch.Tensor):
res = res.cpu().numpy()
res2 = res2.cpu().numpy()
assert (res == res2).all()
else:
assert res == res2
np_y_preds = to_numpy_multilabel(y_preds)
np_y_true = to_numpy_multilabel(y_true)
assert re._type == "multilabel"
res = res if average else res.mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert recall_score(np_y_true, np_y_preds,
average="samples") == pytest.approx(res)
metric_devices = ["cpu"]
if device.type != "xla":
metric_devices.append(idist.device())
for _ in range(2):
for metric_device in metric_devices:
_test(average=True, n_epochs=1, metric_device=metric_device)
_test(average=True, n_epochs=2, metric_device=metric_device)
_test(average=False, n_epochs=1, metric_device=metric_device)
_test(average=False, n_epochs=2, metric_device=metric_device)
re1 = Recall(is_multilabel=True, average=True)
re2 = Recall(is_multilabel=True, average=False)
assert re1._updated is False
assert re2._updated is False
y_pred = torch.randint(0, 2, size=(10, 4, 20, 23))
y = torch.randint(0, 2, size=(10, 4, 20, 23)).long()
re1.update((y_pred, y))
re2.update((y_pred, y))
assert re1._updated is True
assert re2._updated is True
assert re1.compute() == pytest.approx(re2.compute().mean().item())
def Fbeta(beta,
average=True,
precision=None,
recall=None,
output_transform=None,
device=None):
"""Calculates F-beta score
Args:
beta (float): weight of precision in harmonic mean
average (bool, optional): if True, F-beta score is computed as the unweighted average (across all classes
in multiclass case), otherwise, returns a tensor with F-beta score for each class in multiclass case.
precision (Precision, optional): precision object metric with `average=False` to compute F-beta score
recall (Precision, optional): recall object metric with `average=False` to compute F-beta score
output_transform (callable, optional): a callable that is used to transform the
:class:`~ignite.engine.Engine`'s `process_function`'s output into the
form expected by the metric. It is used only if precision or recall are not provided.
device (str of torch.device, optional): device specification in case of distributed computation usage.
In most of the cases, it can be defined as "cuda:local_rank" or "cuda"
if already set `torch.cuda.set_device(local_rank)`. By default, if a distributed process group is
initialized and available, device is set to `cuda`.
Returns:
MetricsLambda, F-beta metric
"""
if not (beta > 0):
raise ValueError(
"Beta should be a positive integer, but given {}".format(beta))
if precision is not None and output_transform is not None:
raise ValueError(
"If precision argument is provided, output_transform should be None"
)
if recall is not None and output_transform is not None:
raise ValueError(
"If recall argument is provided, output_transform should be None")
if precision is None:
precision = Precision(output_transform=(lambda x: x) if
output_transform is None else output_transform,
average=False,
device=device)
elif precision._average:
raise ValueError("Input precision metric should have average=False")
if recall is None:
recall = Recall(output_transform=(lambda x: x)
if output_transform is None else output_transform,
average=False,
device=device)
elif recall._average:
raise ValueError("Input recall metric should have average=False")
fbeta = (1.0 + beta**2) * precision * recall / (beta**2 * precision +
recall + 1e-15)
if average:
fbeta = fbeta.mean().item()
return fbeta
def run_training(model, train, valid, optimizer, loss, lr_find=False):
print_file(f'Experiment: {rcp.experiment}\nDescription:{rcp.description}',
f'{rcp.base_path}description.txt')
print_file(model, f'{rcp.models_path}model.txt')
print_file(get_transforms(), f'{rcp.models_path}transform_{rcp.stage}.txt')
# Data
train.transform = get_transforms()
valid.transform = get_transforms()
train.save_csv(f'{rcp.base_path}train_df_{rcp.stage}.csv')
valid.save_csv(f'{rcp.base_path}valid_df_{rcp.stage}.csv')
train_loader = DataLoader(train,
batch_size=rcp.bs,
num_workers=8,
shuffle=rcp.shuffle_batch)
valid_loader = DataLoader(valid,
batch_size=rcp.bs,
num_workers=8,
shuffle=rcp.shuffle_batch)
if lr_find: lr_finder(model, optimizer, loss, train_loader, valid_loader)
one_batch = next(iter(train_loader))
dot = make_dot(model(one_batch[0].to(cfg.device)),
params=dict(model.named_parameters()))
dot.render(f'{rcp.models_path}graph', './', format='png', cleanup=True)
summary(model,
one_batch[0].shape[-3:],
batch_size=rcp.bs,
device=cfg.device,
to_file=f'{rcp.models_path}summary_{rcp.stage}.txt')
# Engines
trainer = create_supervised_trainer(model,
optimizer,
loss,
device=cfg.device)
t_evaluator = create_supervised_evaluator(model,
metrics={
'accuracy':
Accuracy(),
'nll':
Loss(loss),
'precision':
Precision(average=True),
'recall':
Recall(average=True),
'topK':
TopKCategoricalAccuracy()
},
device=cfg.device)
v_evaluator = create_supervised_evaluator(
model,
metrics={
'accuracy':
Accuracy(),
'nll':
Loss(loss),
'precision_avg':
Precision(average=True),
'recall_avg':
Recall(average=True),
'topK':
TopKCategoricalAccuracy(),
'conf_mat':
ConfusionMatrix(num_classes=len(valid.classes), average=None),
},
device=cfg.device)
# Tensorboard
tb_logger = TensorboardLogger(log_dir=f'{rcp.tb_log_path}{rcp.stage}')
tb_writer = tb_logger.writer
tb_logger.attach(trainer,
log_handler=OptimizerParamsHandler(optimizer, "lr"),
event_name=Events.EPOCH_STARTED)
tb_logger.attach(trainer,
log_handler=WeightsHistHandler(model),
event_name=Events.EPOCH_COMPLETED)
tb_logger.attach(trainer,
log_handler=WeightsScalarHandler(model),
event_name=Events.ITERATION_COMPLETED)
tb_logger.attach(trainer,
log_handler=GradsScalarHandler(model),
event_name=Events.ITERATION_COMPLETED)
tb_logger.attach(trainer,
log_handler=GradsHistHandler(model),
event_name=Events.EPOCH_COMPLETED)
@trainer.on(Events.EPOCH_COMPLETED)
def tb_and_log_training_stats(engine):
t_evaluator.run(train_loader)
v_evaluator.run(valid_loader)
tb_and_log_train_valid_stats(engine, t_evaluator, v_evaluator,
tb_writer)
@trainer.on(
Events.ITERATION_COMPLETED(every=int(1 + len(train_loader) / 100)))
def print_dash(engine):
print('-', sep='', end='', flush=True)
if cfg.show_batch_images:
@trainer.on(Events.STARTED)
def show_batch_images(engine):
imgs, lbls = next(iter(train_loader))
denormalize = DeNormalize(**rcp.transforms.normalize)
for i in range(len(imgs)):
imgs[i] = denormalize(imgs[i])
imgs = imgs.to(cfg.device)
grid = thv.utils.make_grid(imgs)
tb_writer.add_image('images', grid, 0)
tb_writer.add_graph(model, imgs)
tb_writer.flush()
if cfg.show_top_losses:
@trainer.on(Events.COMPLETED)
def show_top_losses(engine, k=6):
nll_loss = nn.NLLLoss(reduction='none')
df = predict_dataset(model,
valid,
nll_loss,
transform=None,
bs=rcp.bs,
device=cfg.device)
df.sort_values('loss', ascending=False, inplace=True)
df.reset_index(drop=True, inplace=True)
for i, row in df.iterrows():
img = cv2.imread(str(row['fname']))
img = th.as_tensor(img.transpose(2, 0, 1)) # #CHW
tag = f'TopLoss_{engine.state.epoch}/{row.loss:.4f}/{row.target}/{row.pred}/{row.pred2}'
tb_writer.add_image(tag, img, 0)
if i >= k - 1: break
tb_writer.flush()
if cfg.tb_projector:
images, labels = train.select_n_random(250)
# get the class labels for each image
class_labels = [train.classes[lab] for lab in labels]
# log embeddings
features = images.view(-1, images.shape[-1] * images.shape[-2])
tb_writer.add_embedding(features,
metadata=class_labels,
label_img=images)
if cfg.log_pr_curve:
@trainer.on(Events.COMPLETED)
def log_pr_curve(engine):
"""
1. gets the probability predictions in a test_size x num_classes Tensor
2. gets the preds in a test_size Tensor
takes ~10 seconds to run
"""
class_probs = []
class_preds = []
with th.no_grad():
for data in valid_loader:
imgs, lbls = data
imgs, lbls = imgs.to(cfg.device), lbls.to(cfg.device)
output = model(imgs)
class_probs_batch = [
th.softmax(el, dim=0) for el in output
]
_, class_preds_batch = th.max(output, 1)
class_probs.append(class_probs_batch)
class_preds.append(class_preds_batch)
test_probs = th.cat([th.stack(batch) for batch in class_probs])
test_preds = th.cat(class_preds)
for i in range(len(valid.classes)):
""" Takes in a "class_index" from 0 to 9 and plots the corresponding precision-recall curve"""
tensorboard_preds = test_preds == i
tensorboard_probs = test_probs[:, i]
tb_writer.add_pr_curve(f'{rcp.stage}/{valid.classes[i]}',
tensorboard_preds,
tensorboard_probs,
global_step=engine.state.epoch,
num_thresholds=127)
tb_writer.flush()
print()
if cfg.lr_scheduler:
# lr_scheduler = ReduceLROnPlateau(optimizer, 'min', patience=5, factor=.5, min_lr=1e-7, verbose=True)
# v_evaluator.add_event_handler(Events.EPOCH_COMPLETED, lambda engine: lr_scheduler.step(v_evaluator.state.metrics['nll']))
lr_scheduler = DelayedCosineAnnealingLR(optimizer, 10, 5)
trainer.add_event_handler(
Events.EPOCH_COMPLETED,
lambda engine: lr_scheduler.step(trainer.state.epoch))
if cfg.early_stopping:
def score_function(engine):
score = -1 * round(engine.state.metrics['nll'], 5)
# score = engine.state.metrics['accuracy']
return score
es_handler = EarlyStopping(patience=10,
score_function=score_function,
trainer=trainer)
v_evaluator.add_event_handler(Events.COMPLETED, es_handler)
if cfg.save_last_checkpoint:
@trainer.on(Events.EPOCH_COMPLETED(every=1))
def save_last_checkpoint(engine):
checkpoint = {}
objects = {'model': model, 'optimizer': optimizer}
if cfg.lr_scheduler: objects['lr_scheduler'] = lr_scheduler
for k, obj in objects.items():
checkpoint[k] = obj.state_dict()
th.save(checkpoint,
f'{rcp.models_path}last_{rcp.stage}_checkpoint.pth')
if cfg.save_best_checkpoint:
def score_function(engine):
score = -1 * round(engine.state.metrics['nll'], 5)
# score = engine.state.metrics['accuracy']
return score
objects = {'model': model, 'optimizer': optimizer}
if cfg.lr_scheduler: objects['lr_scheduler'] = lr_scheduler
save_best = Checkpoint(
objects,
DiskSaver(f'{rcp.models_path}',
require_empty=False,
create_dir=True),
n_saved=4,
filename_prefix=f'best_{rcp.stage}',
score_function=score_function,
score_name='val_loss',
global_step_transform=global_step_from_engine(trainer))
v_evaluator.add_event_handler(Events.EPOCH_COMPLETED(every=1),
save_best)
load_checkpoint = False
if load_checkpoint:
resume_epoch = 6
cp = f'{rcp.models_path}last_{rcp.stage}_checkpoint.pth'
obj = th.load(f'{cp}')
Checkpoint.load_objects(objects, obj)
@trainer.on(Events.STARTED)
def resume_training(engine):
engine.state.iteration = (resume_epoch - 1) * len(
engine.state.dataloader)
engine.state.epoch = resume_epoch - 1
if cfg.save_confusion_matrix:
@trainer.on(Events.STARTED)
def init_best_loss(engine):
engine.state.metrics['best_loss'] = 1e99
@trainer.on(Events.EPOCH_COMPLETED)
def confusion_matric(engine):
if engine.state.metrics['best_loss'] > v_evaluator.state.metrics[
'nll']:
engine.state.metrics['best_loss'] = v_evaluator.state.metrics[
'nll']
cm = v_evaluator.state.metrics['conf_mat']
cm_df = pd.DataFrame(cm.numpy(),
index=valid.classes,
columns=valid.classes)
pretty_plot_confusion_matrix(
cm_df,
f'{rcp.results_path}cm_{rcp.stage}_{trainer.state.epoch}.png',
False)
if cfg.log_stats:
class Hook:
def __init__(self, module):
self.name = module[0]
self.hook = module[1].register_forward_hook(self.hook_fn)
self.stats_mean = 0
self.stats_std = 0
def hook_fn(self, module, input, output):
self.stats_mean = output.mean()
self.stats_std = output.std()
def close(self):
self.hook.remove()
hookF = [Hook(layer) for layer in list(model.cnn.named_children())]
@trainer.on(Events.ITERATION_COMPLETED)
def log_stats(engine):
std = {}
mean = {}
for hook in hookF:
tb_writer.add_scalar(f'std/{hook.name}', hook.stats_std,
engine.state.iteration)
tb_writer.add_scalar(f'mean/{hook.name}', hook.stats_mean,
engine.state.iteration)
cfg.save_yaml()
rcp.save_yaml()
print(f'# batches: train: {len(train_loader)}, valid: {len(valid_loader)}')
trainer.run(data=train_loader, max_epochs=rcp.max_epochs)
tb_writer.close()
tb_logger.close()
return model
def run(tb, vb, lr, epochs, writer):
device = os.environ['main-device']
logging.info('Training program start!')
logging.info('Configuration:')
logging.info('\n'+json.dumps(INFO, indent=2))
# ------------------------------------
# 1. Define dataloader
train_loader, train4val_loader, val_loader, num_of_images, mapping = get_dataloaders(tb, vb)
weights = (1/num_of_images)/((1/num_of_images).sum().item())
# weights = (1/num_of_images)/(1/num_of_images + 1/(num_of_images.sum().item()-num_of_images))
weights = weights.to(device=device)
# ------------------------------------
# 2. Define model
model = EfficientNet.from_pretrained('efficientnet-b5', num_classes=INFO['dataset-info']['num-of-classes'])
model = carrier(model)
# ------------------------------------
# 3. Define optimizer
optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs)
ignite_scheduler = LRScheduler(scheduler)
# ------------------------------------
# 4. Define metrics
train_metrics = {
'accuracy': Accuracy(),
'loss': Loss(CrossEntropywithLS(weight=weights)),
'precision_recall': MetricsLambda(PrecisionRecallTable, Precision(), Recall(), train_loader.dataset.classes),
'cmatrix': MetricsLambda(CMatrixTable, ConfusionMatrix(INFO['dataset-info']['num-of-classes']), train_loader.dataset.classes)
}
# ------------------------------------
# 5. Create trainer
trainer = create_supervised_trainer(model, optimizer, CrossEntropywithLS(weight=weights), device=device)
# ------------------------------------
# 6. Create evaluator
train_evaluator = create_supervised_evaluator(model, metrics=train_metrics, device=device)
desc = 'ITERATION - loss: {:.4f}'
pbar = tqdm(
initial=0, leave=False, total=len(train_loader),
desc=desc.format(0)
)
# ------------------------------------
# 7. Create event hooks
# Update process bar on each iteration completed.
@trainer.on(Events.ITERATION_COMPLETED)
def log_training_loss(engine):
log_interval = 1
iter = (engine.state.iteration - 1) % len(train_loader) + 1
if iter % log_interval == 0:
pbar.desc = desc.format(engine.state.output)
pbar.update(log_interval)
# Refresh Process bar.
@trainer.on(Events.EPOCH_COMPLETED)
def refresh_pbar(engine):
print ('Epoch {} completed!'.format(engine.state.epoch))
pbar.refresh()
pbar.n = pbar.last_print_n = 0
# Compute metrics on train data on each epoch completed.
# cpe = CustomPeriodicEvent(n_epochs=50)
# cpe.attach(trainer)
# @trainer.on(cpe.Events.EPOCHS_50_COMPLETED)
def log_training_results(engine):
pbar.refresh()
print ('Checking on training set.')
train_evaluator.run(train4val_loader)
metrics = train_evaluator.state.metrics
avg_accuracy = metrics['accuracy']
avg_loss = metrics['loss']
precision_recall = metrics['precision_recall']
cmatrix = metrics['cmatrix']
prompt = """
Id: {}
Training Results - Epoch: {}
Avg accuracy: {:.4f}
Avg loss: {:.4f}
precision_recall: \n{}
confusion matrix: \n{}
""".format(os.environ['run-id'],engine.state.epoch,avg_accuracy,avg_loss,precision_recall['pretty'],cmatrix['pretty'])
tqdm.write(prompt)
logging.info('\n'+prompt)
writer.add_text(os.environ['run-id'], prompt, engine.state.epoch)
writer.add_scalars('Aggregate/Acc', {'Train Acc': avg_accuracy}, engine.state.epoch)
writer.add_scalars('Aggregate/Loss', {'Train Loss': avg_loss}, engine.state.epoch)
# pbar.n = pbar.last_print_n = 0
cpe = CustomPeriodicEvent(n_epochs=50)
cpe.attach(trainer)
# @trainer.on(cpe.Events.EPOCHS_50_COMPLETED)
trainer.add_event_handler(cpe.Events.EPOCHS_50_COMPLETED, log_training_results)
trainer.add_event_handler(Events.STARTED, log_training_results)
# Save model ever N epoch.
save_model_handler = ModelCheckpoint(os.environ['savedir'], '', save_interval=10, n_saved=2)
trainer.add_event_handler(Events.EPOCH_COMPLETED, save_model_handler, {'model': model})
# Update learning-rate due to scheduler.
trainer.add_event_handler(Events.EPOCH_STARTED, ignite_scheduler)
# ------------------------------------
# Run
trainer.run(train_loader, max_epochs=epochs)
pbar.close()
def test_pytorch_operators():
def _test(composed_metric, metric_name, compute_true_value_fn):
metrics = {
metric_name: composed_metric,
}
y_pred = torch.rand(15, 10, 5).float()
y = torch.randint(0, 5, size=(15, 10)).long()
def update_fn(engine, batch):
y_pred, y = batch
return y_pred, y
validator = Engine(update_fn)
for name, metric in metrics.items():
metric.attach(validator, name)
def data(y_pred, y):
for i in range(y_pred.shape[0]):
yield (y_pred[i], y[i])
d = data(y_pred, y)
state = validator.run(d, max_epochs=1, epoch_length=y_pred.shape[0])
assert set(state.metrics.keys()) == set([metric_name])
np_y_pred = np.argmax(y_pred.numpy(), axis=-1).ravel()
np_y = y.numpy().ravel()
assert state.metrics[metric_name] == approx(
compute_true_value_fn(np_y_pred, np_y))
precision_1 = Precision(average=False)
precision_2 = Precision(average=False)
norm_summed_precision = (precision_1 + precision_2).norm(p=10)
def compute_true_norm_summed_precision(y_pred, y):
p1 = precision_score(y, y_pred, average=None)
p2 = precision_score(y, y_pred, average=None)
return np.linalg.norm(p1 + p2, ord=10)
_test(norm_summed_precision,
"mean summed precision",
compute_true_value_fn=compute_true_norm_summed_precision)
precision = Precision(average=False)
recall = Recall(average=False)
sum_precision_recall = (precision + recall).sum()
def compute_sum_precision_recall(y_pred, y):
p = precision_score(y, y_pred, average=None)
r = recall_score(y, y_pred, average=None)
return np.sum(p + r)
_test(sum_precision_recall,
"sum precision recall",
compute_true_value_fn=compute_sum_precision_recall)
precision = Precision(average=False)
recall = Recall(average=False)
f1 = (precision * recall * 2 / (precision + recall + 1e-20)).mean()
def compute_f1(y_pred, y):
f1 = f1_score(y, y_pred, average="macro")
return f1
_test(f1, "f1", compute_true_value_fn=compute_f1)
def train(lstm):
device = "cpu"
train_loader = zip(train_iter, train_logits_iter)
test_loader = zip(test_iter, test_logits_iter)
optimizer = Adam(lstm.parameters(), 0.001)
ce_loss = torch.nn.CrossEntropyLoss()
running_avgs = OrderedDict()
def step(engine, batch):
x, teacher_logits = batch
teacher_logits = teacher_logits[0]
text = x.text.to(device)
aspect = x.aspect.to(device)
y_true = x.label.to(device)
n_batch = len(aspect)
optimizer.zero_grad()
y_pred = lstm(text.transpose(0, 1), aspect)
distill_loss = distillation_loss(y_pred,
y_true,
teacher_logits=teacher_logits)
distill_loss.backward()
optimizer.step()
return {'distill_loss': distill_loss.item()}
metrics = {
'avg_accuracy': Accuracy(),
'avg_precision': Precision(average=True),
'avg_recall': Recall(average=True)
}
trainer = Engine(step)
train_evaluator = custom_supervised_evaluator(lstm,
metrics=metrics,
device=device,
loss=ce_loss)
val_evaluator = custom_supervised_evaluator(lstm,
metrics=metrics,
device=device,
loss=ce_loss)
@trainer.on(Events.EPOCH_COMPLETED)
def compute_and_display_offline_train_metrics(engine):
epoch = engine.state.epoch
metrics = train_evaluator.run(train_loader).metrics
print(
"Training Results - Epoch: {} Accuracy: {:.4f} | Precision: {:.4f} | Recall: {:.4f}"
.format(engine.state.epoch, metrics['avg_accuracy'],
metrics['avg_precision'], metrics['avg_recall']))
@trainer.on(Events.EPOCH_COMPLETED)
def compute_and_display_val_metrics(engine):
epoch = engine.state.epoch
metrics = val_evaluator.run(test_loader).metrics
print(
"Validation Results - Epoch: {} Accuracy: {:.4f} | Precision: {:.4f} | Recall: {:.4f}"
.format(engine.state.epoch, metrics['avg_accuracy'],
metrics['avg_precision'], metrics['avg_recall']))
checkpoint_handler = ModelCheckpoint("./",
'checkpoint',
save_interval=3,
n_saved=10,
require_empty=False,
create_dir=True)
progress_bar = Progbar(loader=train_loader, metrics=running_avgs)
trainer.add_event_handler(event_name=Events.EPOCH_COMPLETED,
handler=checkpoint_handler,
to_save={'net': lstm})
trainer.add_event_handler(event_name=Events.ITERATION_COMPLETED,
handler=progress_bar)
trainer.run(train_loader, max_epochs=50)
def test_indexing_metric():
def _test(ignite_metric,
sklearn_metic,
sklearn_args,
index,
num_classes=5):
y_pred = torch.rand(15, 10, num_classes).float()
y = torch.randint(0, num_classes, size=(15, 10)).long()
def update_fn(engine, batch):
y_pred, y = batch
return y_pred, y
metrics = {
"metric": ignite_metric[index],
"metric_wo_index": ignite_metric
}
validator = Engine(update_fn)
for name, metric in metrics.items():
metric.attach(validator, name)
def data(y_pred, y):
for i in range(y_pred.shape[0]):
yield (y_pred[i], y[i])
d = data(y_pred, y)
state = validator.run(d, max_epochs=1, epoch_length=y_pred.shape[0])
sklearn_output = sklearn_metic(
y.view(-1).numpy(),
y_pred.view(-1, num_classes).argmax(dim=1).numpy(), **sklearn_args)
assert (state.metrics["metric_wo_index"][index] ==
state.metrics["metric"]).all()
assert np.allclose(state.metrics["metric"].numpy(), sklearn_output)
num_classes = 5
labels = list(range(0, num_classes, 2))
_test(Precision(),
precision_score, {
"labels": labels,
"average": None
},
index=labels)
labels = list(range(num_classes - 1, 0, -2))
_test(Precision(),
precision_score, {
"labels": labels,
"average": None
},
index=labels)
labels = [1]
_test(Precision(),
precision_score, {
"labels": labels,
"average": None
},
index=labels)
labels = list(range(0, num_classes, 2))
_test(Recall(),
recall_score, {
"labels": labels,
"average": None
},
index=labels)
labels = list(range(num_classes - 1, 0, -2))
_test(Recall(),
recall_score, {
"labels": labels,
"average": None
},
index=labels)
labels = [1]
_test(Recall(),
recall_score, {
"labels": labels,
"average": None
},
index=labels)
# np.ix_ is used to allow for a 2D slice of a matrix. This is required to get accurate result from
# ConfusionMatrix. ConfusionMatrix must be sliced the same row-wise and column-wise.
labels = list(range(0, num_classes, 2))
_test(ConfusionMatrix(num_classes),
confusion_matrix, {"labels": labels},
index=np.ix_(labels, labels))
labels = list(range(num_classes - 1, 0, -2))
_test(ConfusionMatrix(num_classes),
confusion_matrix, {"labels": labels},
index=np.ix_(labels, labels))
labels = [1]
_test(ConfusionMatrix(num_classes),
confusion_matrix, {"labels": labels},
index=np.ix_(labels, labels))
def run(loop: Loop):
seed_everything(42)
setup_cudnn_reproducibility(True, False)
train_ds, valid_ds = get_train_test_datasets("data/cifar")
model = auto_model(get_model())
train_loader = auto_dataloader(
train_ds,
batch_size=512,
shuffle=True,
drop_last=True,
num_workers=4,
)
valid_loader = auto_dataloader(
valid_ds,
batch_size=512,
num_workers=4,
shuffle=False,
)
optim = SGD(model.parameters(), lr=0.4, momentum=0.9)
scheduler = OneCycleLR(optim,
max_lr=1,
epochs=NUM_EPOCHS,
steps_per_epoch=len(train_loader))
criterion = CrossEntropyLoss()
precision = Precision(average=False)
recall = Recall(average=False)
# Ignite metrics are combinable
f1 = (precision * recall * 2 / (precision + recall)).mean()
accuracy = Accuracy()
# We are attaching metrics to automatically reset
loop.attach(
# Loop manages train/eval modes, device and requires_grad of attached `nn.Module`s
criterion=criterion,
# This criterion doesn't have any state or attribute tensors
# So it's attachment doesn't introduce any behavior
model=model,
# Loop saves state of all attached objects having state_dict()/load_state_dict() methods
# to checkpoints
optimizer=optim,
scheduler=scheduler,
)
def train(loop: Loop):
for _ in loop.iterate_epochs(NUM_EPOCHS):
for x, y in loop.iterate_dataloader(train_loader, mode="train"):
y_pred_logits = model(x)
loss: torch.Tensor = criterion(y_pred_logits, y)
loop.backward(loss)
# Makes optimizer step and also
# zeroes grad after (default)
loop.optimizer_step(optim, zero_grad=True)
# Here we call scheduler.step() every iteration
# because we have one-cycle scheduler
# we also can call it after all dataloader loop
# if it's som usual scheduler
scheduler.step()
# Log learning rate. All metrics are written to tensorboard
# with specified names
# If iteration='auto' (default) its determined based on where the call is
# performed. Here it will be batches
loop.metrics.log("lr",
scheduler.get_last_lr()[0],
iteration="auto")
# Loop disables gradients and calls Module.eval() inside loop
# for all attached modules when mode="valid" (default)
for x, y in loop.iterate_dataloader(valid_loader, mode="valid"):
y_pred_logits: torch.Tensor = model(x)
y_pred = to_onehot(y_pred_logits.argmax(dim=-1),
num_classes=10)
precision.update((y_pred, y))
recall.update((y_pred, y))
accuracy.update((y_pred, y))
# This metrics will be epoch metrics because they are called outside
# dataloader loop
# Here we logging metric without resetting it
loop.metrics.log("valid/precision", precision.compute().mean())
loop.metrics.log("valid/recall", recall.compute().mean())
# .log() method above accepts values (tensors, floats, np.array's)
# .consume() accepts Metric object. It resets it after logging
loop.metrics.consume("valid/f1", f1)
loop.metrics.consume("valid/accuracy", accuracy)
loop.run(train)
seq_len = text[1].cpu().numpy()
seq_len[::-1].sort()
softmax = nn.Softmax(dim=1)
y_pred = classifier(x, seq_len)
y_pred = softmax(y_pred)
return y_pred.cpu(), y.squeeze().cpu()
trainer = Engine(training_update_function)
evaluator = create_supervised_evaluator(model=classifier,
inference_fn=inference_function,
metrics={
"loss": Loss(loss_fn),
"acc": CategoricalAccuracy(),
"prec": Precision(),
"rec": Recall()
})
checkpoint = ModelCheckpoint(ARGS.model_dir,
"sentiment",
save_interval=ARGS.save_interval,
n_saved=5,
create_dir=True,
require_empty=False)
loader = ModelLoader(classifier, ARGS.model_dir, "sentiment")
model_name = model_config["model"].split(".")[1]
# Event handlers
trainer.add_event_handler(Events.STARTED, loader, model_name)
trainer.add_event_handler(Events.ITERATION_COMPLETED, checkpoint,
{model_name: classifier.module})
trainer.add_event_handler(Events.COMPLETED, checkpoint,
def test_multilabel_wrong_inputs():
re = Recall(average=True, is_multilabel=True)
with pytest.raises(ValueError):
# incompatible shapes
re.update((torch.randint(0, 2, size=(10, )),
torch.randint(0, 2, size=(10, )).long()))
with pytest.raises(ValueError):
# incompatible y_pred
re.update((torch.rand(10, 5), torch.randint(0, 2,
size=(10, 5)).long()))
with pytest.raises(ValueError):
# incompatible y
re.update((torch.randint(0, 5, size=(10, 5, 6)), torch.rand(10)))
with pytest.raises(ValueError):
# incompatible shapes between two updates
re.update((torch.randint(0, 2, size=(20, 5)),
torch.randint(0, 2, size=(20, 5)).long()))
re.update((torch.randint(0, 2, size=(20, 6)),
torch.randint(0, 2, size=(20, 6)).long()))
def train(self, config, **kwargs):
"""Trains a given model specified in the config file or passed as the --model parameter.
All options in the config file can be overwritten as needed by passing --PARAM
Options with variable lengths ( e.g., kwargs can be passed by --PARAM '{"PARAM1":VAR1, "PARAM2":VAR2}'
:param config: yaml config file
:param **kwargs: parameters to overwrite yaml config
"""
config_parameters = utils.parse_config_or_kwargs(config, **kwargs)
outputdir = os.path.join(
config_parameters['outputpath'], config_parameters['model'],
"{}_{}".format(
datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%m'),
uuid.uuid1().hex))
# Create base dir
Path(outputdir).mkdir(exist_ok=True, parents=True)
logger = utils.getfile_outlogger(os.path.join(outputdir, 'train.log'))
logger.info("Storing files in {}".format(outputdir))
# utils.pprint_dict
utils.pprint_dict(config_parameters, logger.info)
logger.info("Running on device {}".format(DEVICE))
labels_df = pd.read_csv(config_parameters['label'],
sep='\s+').convert_dtypes()
# In case of ave dataset where index is int, we change the
# absolute name to relname
if not np.all(labels_df['filename'].str.isnumeric()):
labels_df.loc[:, 'filename'] = labels_df['filename'].apply(
os.path.basename)
encoder = utils.train_labelencoder(labels=labels_df['event_labels'])
# These labels are useless, only for mode == stratified
label_array, _ = utils.encode_labels(labels_df['event_labels'],
encoder)
if 'cv_label' in config_parameters:
cv_df = pd.read_csv(config_parameters['cv_label'],
sep='\s+').convert_dtypes()
if not np.all(cv_df['filename'].str.isnumeric()):
cv_df.loc[:, 'filename'] = cv_df['filename'].apply(
os.path.basename)
train_df = labels_df
logger.info(
f"Using CV labels from {config_parameters['cv_label']}")
else:
train_df, cv_df = utils.split_train_cv(
labels_df, y=label_array, **config_parameters['data_args'])
if 'cv_data' in config_parameters:
cv_data = config_parameters['cv_data']
logger.info(f"Using CV data {config_parameters['cv_data']}")
else:
cv_data = config_parameters['data']
train_label_array, _ = utils.encode_labels(train_df['event_labels'],
encoder)
cv_label_array, _ = utils.encode_labels(cv_df['event_labels'], encoder)
transform = utils.parse_transforms(config_parameters['transforms'])
utils.pprint_dict({'Classes': encoder.classes_},
logger.info,
formatter='pretty')
torch.save(encoder, os.path.join(outputdir, 'run_encoder.pth'))
torch.save(config_parameters, os.path.join(outputdir,
'run_config.pth'))
logger.info("Transforms:")
utils.pprint_dict(transform, logger.info, formatter='pretty')
# For Unbalanced Audioset, this is true
if 'sampler' in config_parameters and config_parameters[
'sampler'] == 'MultiBalancedSampler':
# Training sampler that oversamples the dataset to be roughly equally sized
# Calcualtes mean over multiple instances, rather useful when number of classes
# is large
train_sampler = dataset.MultiBalancedSampler(
train_label_array,
num_samples=1 * train_label_array.shape[0],
replacement=True)
sampling_kwargs = {"shuffle": False, "sampler": train_sampler}
elif 'sampler' in config_parameters and config_parameters[
'sampler'] == 'MinimumOccupancySampler':
# Asserts that each "batch" contains at least one instance
train_sampler = dataset.MinimumOccupancySampler(
train_label_array, sampling_mode='same')
sampling_kwargs = {"shuffle": False, "sampler": train_sampler}
else:
sampling_kwargs = {"shuffle": True}
logger.info("Using Sampler {}".format(sampling_kwargs))
trainloader = dataset.getdataloader(
{
'filename': train_df['filename'].values,
'encoded': train_label_array
},
config_parameters['data'],
transform=transform,
batch_size=config_parameters['batch_size'],
colname=config_parameters['colname'],
num_workers=config_parameters['num_workers'],
**sampling_kwargs)
cvdataloader = dataset.getdataloader(
{
'filename': cv_df['filename'].values,
'encoded': cv_label_array
},
cv_data,
transform=None,
shuffle=False,
colname=config_parameters['colname'],
batch_size=config_parameters['batch_size'],
num_workers=config_parameters['num_workers'])
model = getattr(models, config_parameters['model'],
'CRNN')(inputdim=trainloader.dataset.datadim,
outputdim=len(encoder.classes_),
**config_parameters['model_args'])
if 'pretrained' in config_parameters and config_parameters[
'pretrained'] is not None:
models.load_pretrained(model,
config_parameters['pretrained'],
outputdim=len(encoder.classes_))
logger.info("Loading pretrained model {}".format(
config_parameters['pretrained']))
model = model.to(DEVICE)
if config_parameters['optimizer'] == 'AdaBound':
try:
import adabound
optimizer = adabound.AdaBound(
model.parameters(), **config_parameters['optimizer_args'])
except ImportError:
config_parameters['optimizer'] = 'Adam'
config_parameters['optimizer_args'] = {}
else:
optimizer = getattr(
torch.optim,
config_parameters['optimizer'],
)(model.parameters(), **config_parameters['optimizer_args'])
utils.pprint_dict(optimizer, logger.info, formatter='pretty')
utils.pprint_dict(model, logger.info, formatter='pretty')
if DEVICE.type != 'cpu' and torch.cuda.device_count() > 1:
logger.info("Using {} GPUs!".format(torch.cuda.device_count()))
model = torch.nn.DataParallel(model)
criterion = getattr(losses, config_parameters['loss'])().to(DEVICE)
def _train_batch(_, batch):
model.train()
with torch.enable_grad():
optimizer.zero_grad()
output = self._forward(
model, batch) # output is tuple (clip, frame, target)
loss = criterion(*output)
loss.backward()
# Single loss
optimizer.step()
return loss.item()
def _inference(_, batch):
model.eval()
with torch.no_grad():
return self._forward(model, batch)
def thresholded_output_transform(output):
# Output is (clip, frame, target)
y_pred, _, y = output
y_pred = torch.round(y_pred)
return y_pred, y
precision = Precision(thresholded_output_transform, average=False)
recall = Recall(thresholded_output_transform, average=False)
f1_score = (precision * recall * 2 / (precision + recall)).mean()
metrics = {
'Loss': losses.Loss(
criterion), #reimplementation of Loss, supports 3 way loss
'Precision': Precision(thresholded_output_transform),
'Recall': Recall(thresholded_output_transform),
'Accuracy': Accuracy(thresholded_output_transform),
'F1': f1_score,
}
train_engine = Engine(_train_batch)
inference_engine = Engine(_inference)
for name, metric in metrics.items():
metric.attach(inference_engine, name)
def compute_metrics(engine):
inference_engine.run(cvdataloader)
results = inference_engine.state.metrics
output_str_list = [
"Validation Results - Epoch : {:<5}".format(engine.state.epoch)
]
for metric in metrics:
output_str_list.append("{} {:<5.2f}".format(
metric, results[metric]))
logger.info(" ".join(output_str_list))
pbar = ProgressBar(persist=False)
pbar.attach(train_engine)
if 'itercv' in config_parameters and config_parameters[
'itercv'] is not None:
train_engine.add_event_handler(
Events.ITERATION_COMPLETED(every=config_parameters['itercv']),
compute_metrics)
train_engine.add_event_handler(Events.EPOCH_COMPLETED, compute_metrics)
# Default scheduler is using patience=3, factor=0.1
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer, **config_parameters['scheduler_args'])
@inference_engine.on(Events.EPOCH_COMPLETED)
def update_reduce_on_plateau(engine):
logger.info(f"Scheduling epoch {engine.state.epoch}")
val_loss = engine.state.metrics['Loss']
if 'ReduceLROnPlateau' == scheduler.__class__.__name__:
scheduler.step(val_loss)
else:
scheduler.step()
early_stop_handler = EarlyStopping(
patience=config_parameters['early_stop'],
score_function=self._negative_loss,
trainer=train_engine)
inference_engine.add_event_handler(Events.EPOCH_COMPLETED,
early_stop_handler)
if config_parameters['save'] == 'everyepoch':
checkpoint_handler = ModelCheckpoint(outputdir,
'run',
n_saved=5,
require_empty=False)
train_engine.add_event_handler(Events.EPOCH_COMPLETED,
checkpoint_handler, {
'model': model,
})
train_engine.add_event_handler(
Events.ITERATION_COMPLETED(every=config_parameters['itercv']),
checkpoint_handler, {
'model': model,
})
else:
checkpoint_handler = ModelCheckpoint(
outputdir,
'run',
n_saved=1,
require_empty=False,
score_function=self._negative_loss,
global_step_transform=global_step_from_engine(
train_engine), # Just so that model is saved with epoch...
score_name='loss')
inference_engine.add_event_handler(Events.EPOCH_COMPLETED,
checkpoint_handler, {
'model': model,
})
train_engine.run(trainloader, max_epochs=config_parameters['epochs'])
return outputdir
def _test_distrib_integration_multilabel(device):
from ignite.engine import Engine
rank = idist.get_rank()
torch.manual_seed(12)
def _test(average, n_epochs, metric_device):
n_iters = 60
s = 16
n_classes = 7
offset = n_iters * s
y_true = torch.randint(0,
2,
size=(offset * idist.get_world_size(),
n_classes, 6, 8)).to(device)
y_preds = torch.randint(0,
2,
size=(offset * idist.get_world_size(),
n_classes, 6, 8)).to(device)
def update(engine, i):
return (
y_preds[i * s + rank * offset:(i + 1) * s + rank * offset,
...],
y_true[i * s + rank * offset:(i + 1) * s + rank * offset, ...],
)
engine = Engine(update)
re = Recall(average=average, is_multilabel=True, device=metric_device)
re.attach(engine, "re")
data = list(range(n_iters))
engine.run(data=data, max_epochs=n_epochs)
assert "re" in engine.state.metrics
res = engine.state.metrics["re"]
res2 = re.compute()
if isinstance(res, torch.Tensor):
res = res.cpu().numpy()
res2 = res2.cpu().numpy()
assert (res == res2).all()
else:
assert res == res2
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
true_res = recall_score(to_numpy_multilabel(y_true),
to_numpy_multilabel(y_preds),
average="samples" if average else None)
assert pytest.approx(res) == true_res
metric_devices = ["cpu"]
if device.type != "xla":
metric_devices.append(idist.device())
for _ in range(2):
for metric_device in metric_devices:
_test(average=True, n_epochs=1, metric_device=metric_device)
_test(average=True, n_epochs=2, metric_device=metric_device)
if idist.get_world_size() > 1:
with pytest.warns(
RuntimeWarning,
match=
"Precision/Recall metrics do not work in distributed setting when "
"average=False and is_multilabel=True",
):
re = Recall(average=False, is_multilabel=True)
y_pred = torch.randint(0, 2, size=(4, 3, 6, 8))
y = torch.randint(0, 2, size=(4, 3, 6, 8)).long()
re.update((y_pred, y))
re_compute1 = re.compute()
re_compute2 = re.compute()
assert len(re_compute1) == 4 * 6 * 8
assert (re_compute1 == re_compute2).all()
def _test(average):
re = Recall(average=average)
y_pred = torch.rand(20, 6)
y = torch.randint(0, 6, size=(20, )).long()
re.update((y_pred, y))
num_classes = y_pred.shape[1]
np_y_pred = y_pred.argmax(dim=1).numpy().ravel()
np_y = y.numpy().ravel()
assert re._type == "multiclass"
assert isinstance(re.compute(), float if average else torch.Tensor)
re_compute = re.compute() if average else re.compute().numpy()
sk_average_parameter = "macro" if average else None
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
sk_compute = recall_score(
np_y,
np_y_pred,
labels=range(0, num_classes),
average=sk_average_parameter,
)
assert sk_compute == pytest.approx(re_compute)
re.reset()
y_pred = torch.rand(10, 4)
y = torch.randint(0, 4, size=(10, )).long()
re.update((y_pred, y))
num_classes = y_pred.shape[1]
np_y_pred = y_pred.argmax(dim=1).numpy().ravel()
np_y = y.numpy().ravel()
assert re._type == "multiclass"
assert isinstance(re.compute(), float if average else torch.Tensor)
re_compute = re.compute() if average else re.compute().numpy()
sk_average_parameter = "macro" if average else None
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
sk_compute = recall_score(
np_y,
np_y_pred,
labels=range(0, num_classes),
average=sk_average_parameter,
)
assert sk_compute == pytest.approx(re_compute)
# 2-classes
re.reset()
y_pred = torch.rand(10, 2)
y = torch.randint(0, 2, size=(10, )).long()
re.update((y_pred, y))
num_classes = y_pred.shape[1]
np_y_pred = y_pred.argmax(dim=1).numpy().ravel()
np_y = y.numpy().ravel()
assert re._type == "multiclass"
assert isinstance(re.compute(), float if average else torch.Tensor)
re_compute = re.compute() if average else re.compute().numpy()
sk_average_parameter = "macro" if average else None
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
sk_compute = recall_score(
np_y,
np_y_pred,
labels=range(0, num_classes),
average=sk_average_parameter,
)
assert sk_compute == pytest.approx(re_compute)
# Batched Updates
re.reset()
y_pred = torch.rand(100, 3)
y = torch.randint(0, 3, size=(100, )).long()
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
re.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
num_classes = y_pred.shape[1]
np_y = y.numpy().ravel()
np_y_pred = y_pred.argmax(dim=1).numpy().ravel()
assert re._type == "multiclass"
assert isinstance(re.compute(), float if average else torch.Tensor)
re_compute = re.compute() if average else re.compute().numpy()
sk_average_parameter = "macro" if average else None
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
sk_compute = recall_score(
np_y,
np_y_pred,
labels=range(0, num_classes),
average=sk_average_parameter,
)
assert sk_compute == pytest.approx(re_compute)
def _test(average):
re = Recall(average=average)
y_pred = torch.randint(0, 2, size=(10, 12, 10))
y = torch.randint(0, 2, size=(10, 12, 10)).type(torch.LongTensor)
re.update((y_pred, y))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert re._type == 'binary'
assert isinstance(re.compute(), float if average else torch.Tensor)
re_compute = re.compute() if average else re.compute().numpy()
assert recall_score(np_y, np_y_pred,
average='binary') == pytest.approx(re_compute)
re.reset()
y_pred = torch.randint(0, 2, size=(10, 1, 12, 10))
y = torch.randint(0, 2, size=(10, 1, 12, 10)).type(torch.LongTensor)
re.update((y_pred, y))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert re._type == 'binary'
assert isinstance(re.compute(), float if average else torch.Tensor)
re_compute = re.compute() if average else re.compute().numpy()
assert recall_score(np_y, np_y_pred,
average='binary') == pytest.approx(re_compute)
re = Recall(average=average)
# Batched Updates
re.reset()
y_pred = torch.randint(0, 2, size=(100, 12, 10))
y = torch.randint(0, 2, size=(100, 1, 12, 10)).type(torch.LongTensor)
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
re.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().ravel()
assert re._type == 'binary'
assert isinstance(re.compute(), float if average else torch.Tensor)
re_compute = re.compute() if average else re.compute().numpy()
assert recall_score(np_y, np_y_pred,
average='binary') == pytest.approx(re_compute)
# F1 = (precision * recall * 2 / (precision + recall)).mean()
tqdm.write(
"Validation Results - Epoch: {} Avg accuracy: {:.2f} Avg loss: {:.2f}"
.format(engine.state.epoch, avg_accuracy, avg_loss))
pbar.n = pbar.last_print_n = 0
trainer.run(train_loader, max_epochs=args.num_epochs)
pbar.close()
tester = create_supervised_evaluator(model,
metrics={
'accuracy': Accuracy(),
'loss': Loss(criterion),
'pre': Precision(average=True),
'recall': Recall(average=False)
},
device=device)
tester.run(test_loader)
metrics = tester.state.metrics
test_accuracy = metrics['accuracy']
test_loss = metrics['loss']
print("Precision", metrics['pre'])
print("Recall", metrics['recall'])
print("Test Results - Avg accuracy: {:.2f} Avg loss: {:.2f}".format(
test_accuracy, test_loss))
stats = {
'train_accuracy': train_accuracy,
'train_loss': train_loss,
'val_accuracy': val_accuracy,
'val_loss': val_loss,
def _test(average):
re = Recall(average=average)
y_pred = torch.rand(10, 5, 8)
y = torch.randint(0, 5, size=(10, 8)).type(torch.LongTensor)
re.update((y_pred, y))
num_classes = y_pred.shape[1]
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert re._type == 'multiclass'
assert isinstance(re.compute(), float if average else torch.Tensor)
re_compute = re.compute() if average else re.compute().numpy()
sk_average_parameter = 'macro' if average else None
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert recall_score(
np_y, np_y_pred,
average=sk_average_parameter) == pytest.approx(re_compute)
re.reset()
y_pred = torch.rand(15, 10, 8)
y = torch.randint(0, 10, size=(15, 8)).type(torch.LongTensor)
re.update((y_pred, y))
num_classes = y_pred.shape[1]
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
np_y = y.numpy().ravel()
assert re._type == 'multiclass'
assert isinstance(re.compute(), float if average else torch.Tensor)
re_compute = re.compute() if average else re.compute().numpy()
sk_average_parameter = 'macro' if average else None
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
sk_compute = recall_score(np_y,
np_y_pred,
labels=range(0, num_classes),
average=sk_average_parameter)
assert sk_compute == pytest.approx(re_compute)
# Batched Updates
re.reset()
y_pred = torch.rand(100, 8, 12)
y = torch.randint(0, 8, size=(100, 12)).type(torch.LongTensor)
batch_size = 16
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
re.update((y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
num_classes = y_pred.shape[1]
np_y = y.numpy().ravel()
np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
assert re._type == 'multiclass'
assert isinstance(re.compute(), float if average else torch.Tensor)
re_compute = re.compute() if average else re.compute().numpy()
sk_average_parameter = 'macro' if average else None
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
sk_compute = recall_score(np_y,
np_y_pred,
labels=range(0, num_classes),
average=sk_average_parameter)
assert sk_compute == pytest.approx(re_compute)
def test_multilabel_input(average):
re = Recall(average=average, is_multilabel=True)
assert re._updated is False
def _test(y_pred, y, batch_size):
re.reset()
assert re._updated is False
if batch_size > 1:
n_iters = y.shape[0] // batch_size + 1
for i in range(n_iters):
idx = i * batch_size
re.update(
(y_pred[idx:idx + batch_size], y[idx:idx + batch_size]))
else:
re.update((y_pred, y))
np_y_pred = to_numpy_multilabel(y_pred)
np_y = to_numpy_multilabel(y)
assert re._type == "multilabel"
assert re._updated is True
re_compute = re.compute() if average else re.compute().mean().item()
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
assert recall_score(np_y, np_y_pred,
average="samples") == pytest.approx(re_compute)
re1 = Recall(is_multilabel=True, average=True)
re2 = Recall(is_multilabel=True, average=False)
assert re1._updated is False
assert re2._updated is False
re1.update((y_pred, y))
re2.update((y_pred, y))
assert re1._updated is True
assert re2._updated is True
assert re1.compute() == pytest.approx(re2.compute().mean().item())
assert re1._updated is True
assert re2._updated is True
def get_test_cases():
test_cases = [
# Multilabel input data of shape (N, C)
(torch.randint(0, 2,
size=(10, 5)), torch.randint(0, 2,
size=(10, 5)), 1),
(torch.randint(0, 2,
size=(10, 4)), torch.randint(0, 2,
size=(10, 4)), 1),
# updated batches
(torch.randint(0, 2,
size=(50, 5)), torch.randint(0, 2,
size=(50, 5)), 16),
(torch.randint(0, 2,
size=(50, 4)), torch.randint(0, 2,
size=(50, 4)), 16),
# Multilabel input data of shape (N, H, W)
(torch.randint(0, 2, size=(10, 5, 10)),
torch.randint(0, 2, size=(10, 5, 10)), 1),
(torch.randint(0, 2, size=(10, 4, 10)),
torch.randint(0, 2, size=(10, 4, 10)), 1),
# updated batches
(torch.randint(0, 2, size=(50, 5, 10)),
torch.randint(0, 2, size=(50, 5, 10)), 16),
(torch.randint(0, 2, size=(50, 4, 10)),
torch.randint(0, 2, size=(50, 4, 10)), 16),
# Multilabel input data of shape (N, C, H, W, ...)
(torch.randint(0, 2, size=(10, 5, 18, 16)),
torch.randint(0, 2, size=(10, 5, 18, 16)), 1),
(torch.randint(0, 2, size=(10, 4, 20, 23)),
torch.randint(0, 2, size=(10, 4, 20, 23)), 1),
# updated batches
(torch.randint(0, 2, size=(50, 5, 18, 16)),
torch.randint(0, 2, size=(50, 5, 18, 16)), 16),
(torch.randint(0, 2, size=(50, 4, 20, 23)),
torch.randint(0, 2, size=(50, 4, 20, 23)), 16),
# Corner case with all zeros predictions
(torch.zeros(size=(10, 5)), torch.randint(0, 2, size=(10, 5)), 1),
(torch.zeros(size=(10, 4)), torch.randint(0, 2, size=(10, 4)), 1),
]
return test_cases
for _ in range(5):
# check multiple random inputs as random exact occurencies are rare
test_cases = get_test_cases()
for y_pred, y, batch_size in test_cases:
_test(y_pred, y, batch_size)
