def log_epoch_results(self, outputs, stage):
age_pred, age_true, gender_pred, gender_true = self.get_epoch_results(
outputs)
age_labels = [
"0-10", "11-20", "21-30", "31-40", "41-50", "51-60", "60-"
]
cm_age = confusion_matrix_plot_as_array(age_true, age_pred, age_labels)
cm_gender = confusion_matrix_plot_as_array(gender_true, gender_pred,
["Male", "Female"])
# Tensorboard logs
age_accuracy = accuracy(
torch.tensor(age_pred).to(self.device),
(torch.tensor(age_true).to(self.device))).item()
gender_accuracy = accuracy(
torch.tensor(gender_pred).to(self.device),
(torch.tensor(gender_true).to(self.device))).item()
tb = self.logger.experiment
tb.add_scalar(f"{stage}/Age accuracy", age_accuracy,
self.current_epoch)
tb.add_scalar(f"{stage}/Gender accuracy", gender_accuracy,
self.current_epoch)
tb.add_image(f"{stage}/Age",
cm_age,
self.current_epoch,
dataformats="HWC")
tb.add_image(f"{stage}/Gender",
cm_gender,
self.current_epoch,
dataformats="HWC")
def test_accuracy():
pred = torch.tensor([0, 1, 2, 3])
target = torch.tensor([0, 1, 2, 2])
acc = accuracy(pred, target)
assert acc.item() == 0.75
pred = torch.tensor([0, 1, 2, 2])
target = torch.tensor([0, 1, 1, 3])
acc = accuracy(pred, target)
assert acc.item() == 0.50
def training_step(self, batch, batch_idx):
assert self.trainer.max_epochs > self.hparams.burn_in_epochs
if self.trainer.current_epoch < self.hparams.burn_in_epochs:
for param in self.bert.parameters():
param.requires_grad_(False)
for param in self.pooler.parameters():
param.requires_grad_(False)
for param in self.company_layer.parameters():
param.requires_grad_(False)
else:
for param in self.parameters():
param.requires_grad_(True)
input_ids, input_mask, label_ids, doc_ids = batch
logits = self(batch)
loss = cross_entropy(logits,
label_ids,
weight=self.weights.to(self.device))
preds = logits.argmax(dim=1)
batch_dict = {
'loss': loss,
'logits': logits,
'preds': preds,
'labels': label_ids
}
self.log('train_loss', loss)
self.log('train_acc', accuracy(logits, label_ids))
return batch_dict
def training_epoch_end(self, train_outputs):
'''
Log all the values after the end of the epoch.
'''
outputs = [x['result'] for x in train_outputs]
avg_class_loss = torch.stack([x['class_loss'] for x in outputs]).mean()
avg_mask_loss = torch.stack([x['mask_loss'] for x in outputs]).mean()
avg_loss = torch.stack([x['total_loss'] for x in outputs]).mean()
all_predictions = torch.stack([x['predictions']
for x in outputs]).flatten()
all_targets = torch.stack([x['targets'] for x in outputs]).flatten()
class_accuracy = accuracy(all_predictions, all_targets, num_classes=2)
class_f1 = f1_score(all_predictions, all_targets, num_classes=2)
self.log('train_class_loss', avg_class_loss, sync_dist=True)
self.log('train_mask_loss', avg_mask_loss, sync_dist=True)
self.log('train_loss', avg_loss, sync_dist=True)
self.log('train_accuracy',
class_accuracy,
prog_bar=True,
sync_dist=True)
self.log('train_f1', class_f1, sync_dist=True)
def validation_step(self, batch, batch_idx):
x, y = batch
y_hat = self(x)
val_loss = F.cross_entropy(y_hat, y)
val_acc = accuracy(y_hat, y)
self.log('val_loss', val_loss, prog_bar=True)
self.log('val_acc', val_acc, prog_bar=True)
def training_step(self, batch, batch_idx):
x, y = batch
y_hat = self(x)
loss = F.cross_entropy(y_hat, y)
acc = accuracy(y_hat, y)
self.log('loss', loss)
self.log('acc', acc, prog_bar=True)
return loss
def validation_step(self, batch, batch_idx):
x, y = batch
output = self(x)
loss = F.binary_cross_entropy(output, y.float())
acc = accuracy(torch.round(output), y.float())
self.log("val_loss", loss, logger=True)
self.log("val_acc", acc, logger=True)
return {"val_loss": loss, "val_acc": acc}
def validation_epoch_end(self, val_step_outputs):
genre_preds, genre_ys = zip(*val_step_outputs)
genre_preds, genre_ys = torch.cat(genre_preds), torch.cat(genre_ys)
genre_accuracy = accuracy(genre_preds,
genre_ys,
num_classes=self.n_genres,
class_reduction="weighted")
self.log("val_genre_accuracy", genre_accuracy)
def training_step(self, batch, batch_idx):
x, y = batch
features = self.extract_features(x)
y_hat = self.head(features[-1])
loss = F.cross_entropy(y_hat, y)
acc = accuracy(y_hat, y)
self.log('loss', loss)
self.log('acc', acc, prog_bar=True)
return loss
def forward(self, pred: torch.Tensor,
target: torch.Tensor) -> torch.Tensor:
"""
Actual metric computation
Args:
pred: predicted labels
target: ground truth labels
mask: only calculate metrics where mask==1
Return:
A Tensor with the classification score.
"""
return accuracy(pred=pred, target=target, num_classes=self.num_classes)
def validation_step(self, batch, batch_idx):
inputs, targets = batch
loss_function = self.loss
#TODO: JOKER LOSS is probably not needed for validation
output_1, output_2, output_3, output_concat = self(inputs)
outputs_com = output_1 + output_2 + output_3 + output_3 + output_concat
val_loss = loss_function(output_concat, targets)
"""
There is the individual accuracy, and combined accuracy
"""
_, predicted = torch.max(output_concat.data, 1)
_, predicted_com = torch.max(outputs_com.data, 1)
valid_acc = accuracy(predicted, targets)
valid_acc_en = accuracy(predicted_com, targets)
metrics = {
'val_loss': val_loss,
'val_acc': valid_acc,
'val_acc_en': valid_acc_en
}
self.log('val_acc',
valid_acc,
on_step=False,
on_epoch=True,
prog_bar=True,
logger=True)
self.log('val_acc_en',
valid_acc_en,
on_step=False,
on_epoch=True,
prog_bar=True,
logger=True)
return metrics
def forward(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
"""
Actual metric computation
Args:
pred: predicted labels
target: ground truth labels
Return:
A Tensor with the classification score.
"""
return accuracy(pred=pred, target=target,
num_classes=self.num_classes, reduction=self.reduction)
def test_multilabel_accuracy():
# Dense label indicator matrix format
y1 = torch.tensor([[0, 1, 1], [1, 0, 1]])
y2 = torch.tensor([[0, 0, 1], [1, 0, 1]])
assert torch.allclose(accuracy(y1, y2, class_reduction='none'), torch.tensor([2 / 3, 1.]))
assert torch.allclose(accuracy(y1, y1, class_reduction='none'), torch.tensor([1., 1.]))
assert torch.allclose(accuracy(y2, y2, class_reduction='none'), torch.tensor([1., 1.]))
assert torch.allclose(accuracy(y2, torch.logical_not(y2), class_reduction='none'), torch.tensor([0., 0.]))
assert torch.allclose(accuracy(y1, torch.logical_not(y1), class_reduction='none'), torch.tensor([0., 0.]))
with pytest.raises(RuntimeError):
accuracy(y2, torch.zeros_like(y2), class_reduction='none')
def training_step(self, batch, batch_idx):
array_rep, labels_list = batch
atom_features = array_rep['atom_features']
bond_features = array_rep['bond_features']
atom_features = torch.tensor(atom_features, dtype=torch.float)
bond_features = torch.tensor(bond_features, dtype=torch.float)
if self.task == "regression":
labels = torch.tensor(labels_list, dtype=torch.float)
else:
labels = torch.tensor(labels_list, dtype=torch.float)
atom_features = atom_features.to(self.device_ext)
bond_features = bond_features.to(self.device_ext)
labels = labels.to(self.device_ext)
y_pred = self(array_rep, atom_features, bond_features)
if self.task == "regression":
loss = F.mse_loss(y_pred, labels)
# https://github.com/pytorch/ignite/issues/453
var_y = torch.var(labels, unbiased=False)
r2 = 1.0 - F.mse_loss(y_pred, labels, reduction="mean") / var_y
else:
loss = F.binary_cross_entropy_with_logits(y_pred, labels)
y_pred_proba = torch.sigmoid(y_pred)
y_pred = y_pred_proba > 0.5
acc = accuracy(y_pred, labels)
y_pred_proba = y_pred_proba.to('cpu').detach().numpy().tolist()
from sklearn import metrics
try:
rocauc = metrics.roc_auc_score(labels_list, y_pred_proba)
except Exception as es:
rocauc = 0
if self.task == "regression":
ret = {'loss': loss, 'train_r2': r2}
else:
ret = {
'loss': loss,
'train_acc': acc,
'train_rocauc': torch.tensor(rocauc, dtype=torch.float)
}
return ret
def test_step(self, batch: tuple, batch_nb: int, *args, **kwargs) -> dict:
"""
Runs one training step. This usually consists in the forward function followed
by the loss function.
:param batch: The output of your dataloader.
:param batch_nb: Integer displaying which batch this is
Returns:
- dictionary containing the loss and the metrics to be added to the lightning logger.
"""
inputs, targets = batch
model_out = self.forward(**inputs)
loss_val = self.loss(model_out, targets)
# in DP mode (default) make sure if result is scalar, there's another dim in the beginning
if self.trainer.use_dp or self.trainer.use_ddp2:
loss_val = loss_val.unsqueeze(0)
self.log('test_loss',loss_val)
y_hat=model_out['logits']
labels_hat = torch.argmax(y_hat, dim=1)
y=targets['labels']
f1 = metrics.f1_score(labels_hat,y, class_reduction='weighted')
prec =metrics.precision(labels_hat,y, class_reduction='weighted')
recall = metrics.recall(labels_hat,y, class_reduction='weighted')
acc = metrics.accuracy(labels_hat,y, class_reduction='weighted')
# auroc = metrics.multiclass_auroc(labels_hat, y)
self.log('test_batch_prec',prec)
self.log('test_batch_f1',f1)
self.log('test_batch_recall',recall)
self.log('test_batch_weighted_acc', acc)
# self.log('test_batch_auc_roc', auroc)
from pytorch_lightning.metrics.functional import confusion_matrix
# TODO CHANGE THIS
# return (labels_hat, y)
cm = confusion_matrix(preds = labels_hat,target=y,normalize=None, num_classes=50)
# cm = confusion_matrix(preds = labels_hat,target=y,normalize=False, num_classes=len(y.unique()))
self.test_conf_matrices.append(cm)
def validation_step(self, batch, batch_idx):
input_ids, input_mask, label_ids, doc_ids = batch
logits = self(batch)
loss = cross_entropy(logits,
label_ids,
weight=self.weights.to(self.device))
preds = logits.argmax(dim=1)
batch_dict = {
'loss': loss,
'logits': logits,
'preds': preds,
'labels': label_ids
}
self.log('val_loss', loss)
self.log('val_acc', accuracy(logits, label_ids))
return batch_dict
def forward(self, pred: torch.Tensor, target: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
"""
Actual metric computation
Args:
pred: predicted labels
target: ground truth labels
mask: only calculate metrics where mask==1
Return:
A Tensor with the classification score.
"""
# (yuxian): much faster than pytorch-lightning's implementation
if self.reduction == "elementwise_mean":
correct = (pred == target).float() * mask.float()
return correct.sum() / (mask.float().sum() + 1e-10)
mask_fill = (1-mask).bool()
pred = pred.masked_fill(mask=mask_fill, value=-1)
target = target.masked_fill(mask=mask_fill, value=-1)
return accuracy(pred=pred, target=target,
num_classes=self.num_classes, reduction=self.reduction)
def training_step(self, batch, batch_idx):
inputs, targets = batch
loss_function = self.loss
# step 1 (start from fine-grained jigsaw n=8)
inputs1 = jigsaw_generator(inputs, 8)
output_1, _, _, _ = self(inputs1)
loss1 = loss_function(output_1, targets) * 1
# step 2
inputs2 = jigsaw_generator(inputs, 4)
_, output_2, _, _ = self(inputs2)
loss2 = loss_function(output_2, targets) * 1
# step 3
inputs3 = jigsaw_generator(inputs, 2)
_, _, output_3, _ = self(inputs3)
loss3 = loss_function(output_3, targets) * 1
# step 4 whole image, and vanilla loss.
_, _, _, output_concat = self(inputs)
concat_loss = loss_function(output_concat, targets) * 2
train_loss = loss1 + loss2 + loss3 + concat_loss
# accuracy
_, predicted = torch.max(output_concat.data, 1)
train_acc = accuracy(predicted, targets)
metrics = {'loss': train_loss, 'accuracy': train_acc}
self.log('accuracy',
train_acc,
on_step=True,
on_epoch=True,
prog_bar=True,
logger=True)
return metrics
def log_epoch_results(self, outputs, stage):
gender_pred = []
gender_true = []
for i, output in enumerate(outputs):
gender_true.extend(self.tensor_to_array(output["gender"][0]))
gender_pred.extend(self.tensor_to_array(output["gender"][1]))
cm_gender = confusion_matrix_plot_as_array(gender_true, gender_pred,
["Male", "Female"])
# Tensorboard logs
gender_accuracy = accuracy(
torch.tensor(gender_pred).to(self.device),
(torch.tensor(gender_true).to(self.device))).item()
tb = self.logger.experiment
self.log(f"{stage}/Gender accuracy", gender_accuracy)
tb.add_image(f"{stage}/Gender",
cm_gender,
self.current_epoch,
dataformats="HWC")
def validation_epoch_end(self, outputs):
'''
Log all the values after the end of the epoch.
'''
avg_class_loss = torch.stack([x['class_loss'] for x in outputs]).mean()
avg_mask_loss = torch.stack([x['mask_loss'] for x in outputs]).mean()
avg_loss = torch.stack([x['total_loss'] for x in outputs]).mean()
all_predictions = torch.stack([x['predictions']
for x in outputs]).flatten()
all_targets = torch.stack([x['targets'] for x in outputs]).flatten()
class_accuracy = accuracy(all_predictions, all_targets, num_classes=2)
class_f1 = f1_score(all_predictions, all_targets, num_classes=2)
self.log('valid_class_loss', avg_class_loss)
self.log('valid_mask_loss', avg_mask_loss)
self.log('valid_loss', avg_loss)
self.log('valid_accuracy', class_accuracy, prog_bar=True)
self.log('valid_f1', class_f1)
def _get_metrics_at_optimal_cutoff(
self
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Computes the ROC to find the optimal cut-off i.e. the probability threshold for which the
difference between true positive rate and false positive rate is smallest. Then, computes
the false positive rate, false negative rate and accuracy at this threshold (i.e. when the
predicted probability is higher than the threshold the predicted label is 1 otherwise 0).
:returns: Tuple(optimal_threshold, false positive rate, false negative rate, accuracy)
"""
preds, targets = self._get_preds_and_targets()
if torch.unique(targets).numel() == 1:
return torch.tensor(np.nan), torch.tensor(np.nan), torch.tensor(
np.nan), torch.tensor(np.nan)
fpr, tpr, thresholds = roc(preds, targets)
optimal_idx = torch.argmax(tpr - fpr)
optimal_threshold = thresholds[optimal_idx]
acc = accuracy(preds > optimal_threshold, targets)
false_negative_optimal = 1 - tpr[optimal_idx]
false_positive_optimal = fpr[optimal_idx]
return optimal_threshold, false_positive_optimal, false_negative_optimal, acc
def validation_step(self, batch, batch_idx):
data, y = batch
p = self.hparams.shot * self.hparams.test_way
data_shot, data_query = data[:p], data[p:]
proto = self(data_shot)
proto = proto.reshape(self.hparams.shot, self.hparams.test_way,
-1).mean(dim=0)
label = torch.arange(self.hparams.test_way,
device=self.device).repeat(self.hparams.query)
logits = euclidean_metric(self(data_query), proto)
loss = F.cross_entropy(logits, label)
pred = torch.argmax(logits, dim=1)
acc = accuracy(pred, label)
self.log_dict({
'valid_loss': loss,
'val_acc': acc
},
prog_bar=True,
on_step=True)
def validation_epoch_end(self, outputs):
logits = torch.cat([o["logits"] for o in outputs], dim=0)
labels = torch.cat([o["labels"] for o in outputs], dim=0)
self.log(
"val_f1",
f1_score(torch.argmax(logits, dim=1),
labels,
class_reduction="macro").detach().cpu().item(),
prog_bar=True,
)
self.log(
"val_acc",
accuracy(torch.argmax(logits, dim=1),
labels).detach().cpu().item(),
prog_bar=True,
)
self.log(
"val_loss",
self.loss(logits, labels).detach().cpu().item(),
prog_bar=True,
)
def validation_step(self, batch: tuple, batch_nb: int, *args, **kwargs) -> dict:
""" Similar to the training step but with the model in eval mode.
Returns:
- dictionary passed to the validation_end function.
"""
inputs, targets = batch
model_out = self.forward(**inputs)
loss_val = self.loss(model_out, targets)
y = targets["labels"]
y_hat = model_out["logits"]
# acc
labels_hat = torch.argmax(y_hat, dim=1)
val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
val_acc = torch.tensor(val_acc)
if self.on_gpu:
val_acc = val_acc.cuda(loss_val.device.index)
# in DP mode (default) make sure if result is scalar, there's another dim in the beginning
if self.trainer.use_dp or self.trainer.use_ddp2:
loss_val = loss_val.unsqueeze(0)
val_acc = val_acc.unsqueeze(0)
self.log('val_loss',loss_val)
f1 = metrics.f1_score(labels_hat, y,class_reduction='weighted')
prec =metrics.precision(labels_hat, y,class_reduction='weighted')
recall = metrics.recall(labels_hat, y,class_reduction='weighted')
acc = metrics.accuracy(labels_hat, y,class_reduction='weighted')
# auroc = metrics.multiclass_auroc(y_hat,y)
self.log('val_prec',prec)
self.log('val_f1',f1)
self.log('val_recall',recall)
self.log('val_acc_weighted', acc)
def log_epoch_results(self, outputs, stage):
age_pred = []
age_true = []
for i, output in enumerate(outputs):
age_true.extend(self.tensor_to_array(output["age"][0]))
age_pred.extend(self.tensor_to_array(output["age"][1]))
age_labels = [
"0-10", "11-20", "21-30", "31-40", "41-50", "51-60", "60-"
]
cm_age = confusion_matrix_plot_as_array(age_true, age_pred, age_labels)
# Tensorboard logs
age_accuracy = accuracy(
torch.tensor(age_pred).to(self.device),
(torch.tensor(age_true).to(self.device))).item()
tb = self.logger.experiment
self.log(f"{stage}/Age accuracy", age_accuracy)
tb.add_image(f"{stage}/Age",
cm_age,
self.current_epoch,
dataformats="HWC")
def test_multilabel_accuracy():
# Dense label indicator matrix format
y1 = torch.tensor([[0, 1, 1], [1, 0, 1]])
y2 = torch.tensor([[0, 0, 1], [1, 0, 1]])
assert torch.allclose(accuracy(y1, y2, class_reduction='none'),
torch.tensor([2 / 3, 1.]))
assert torch.allclose(accuracy(y1, y1, class_reduction='none'),
torch.tensor([1., 1.]))
assert torch.allclose(accuracy(y2, y2, class_reduction='none'),
torch.tensor([1., 1.]))
assert torch.allclose(
accuracy(y2, torch.logical_not(y2), class_reduction='none'),
torch.tensor([0., 0.]))
assert torch.allclose(
accuracy(y1, torch.logical_not(y1), class_reduction='none'),
torch.tensor([0., 0.]))
# num_classes does not match extracted number from input we expect a warning
with pytest.warns(RuntimeWarning,
match=r'You have set .* number of classes which is'
r' different from predicted (.*) and'
r' target (.*) number of classes'):
_ = accuracy(y2, torch.zeros_like(y2), num_classes=3)
def accuracy_function(self, outputs, ground_truth):
return dict(accuracy=FMC.accuracy(
outputs, ground_truth, num_classes=self.hparams.num_classes))
def calculate_metrics(self, y, y_hat):
loss = F.cross_entropy(y_hat, y)
y_pred = y_hat.argmax(dim=1)
acc = classification.accuracy(y_pred, y)
f1_score = f1(y_pred, y, self.num_classes)
return {"loss": loss, "acc": acc, "f1": f1_score}
def test_step(self, batch, batch_idx):
x, y = batch
output = self(x)
loss = F.binary_cross_entropy(output, y.float())
acc = accuracy(torch.round(output), y.float())
return {"test_loss": loss, "test_acc": acc}
