def __shared_step_op(self, batch, batch_idx, phase, log=True):
img, mask, edge_mask = batch
output = self.forward(img)
loss_matrix = self.criterion(output, mask)
loss = (loss_matrix * (self.edge_weight ** edge_mask)).mean()
output_labels = torch.argmax(output, dim=1).view(-1)
ground_truths = mask.view(-1)
f1 = f1_score(output_labels,
ground_truths,
num_classes=self.n_classes,
class_reduction=self.class_reduction)
precision, recall = precision_recall(output_labels,
ground_truths,
num_classes=self.n_classes,
class_reduction=self.class_reduction)
if self.n_classes == 2:
# use the positive class only for binary case
f1 = f1[-1]
precision = precision[-1]
recall = recall[-1]
if log:
self.log(f"{phase}/loss", loss, prog_bar=True)
self.log(f"{phase}/f1_score", f1, prog_bar=True)
self.log(f"{phase}/precision", precision, prog_bar=False)
self.log(f"{phase}/recall", recall, prog_bar=False)
return {f"{phase}_loss": loss, f"{phase}_f1_score": f1}
def validation_step(self, batch, batch_idx):
self.model.eval()
tokens, intent_idx, entity_idx, text = batch
intent_pred, entity_pred = self.forward(tokens)
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
entity_acc = get_token_accuracy(
entity_pred.argmax(2),
entity_idx,
ignore_index=self.dataset.pad_token_id)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
intent_loss = self.intent_loss_fn(intent_pred, intent_idx.squeeze(1))
entity_loss = self.entity_loss_fn(
entity_pred.transpose(1, 2),
entity_idx.long(),
)
return {
"val_intent_acc": torch.Tensor([intent_acc]),
"val_entity_acc": torch.Tensor([entity_acc]),
"val_intent_f1": intent_f1,
"val_intent_loss": intent_loss,
"val_entity_loss": entity_loss,
"val_loss": intent_loss + entity_loss,
}
def metrics(self, pred, target, num_classes=3, remove_bg=False, is_swnet=False):
if is_swnet:
pred[pred == 2] = 1
target[target == 2] = 1
num_classes = 2
confusion_m = mf.confusion_matrix(pred, target, num_classes=num_classes)
# acc
accuracy = confusion_m.diag().sum() / len(pred)
# kappa
p0 = accuracy
pc = 0
for i in range(confusion_m.shape[0]):
pc = pc + confusion_m[i].sum() * confusion_m[:, i].sum()
pc = pc / len(pred)**2
kc = (p0 - pc) / (1 - pc)
# iou
if remove_bg:
iou = mf.iou(pred, target, num_classes=num_classes, ignore_index=0)
else:
iou = mf.iou(pred, target, num_classes=num_classes)
f1 = mf.f1_score(pred, target, num_classes=num_classes, class_reduction='none')
precision = mf.precision(pred, target, num_classes=num_classes, class_reduction='none')
recall = mf.recall(pred, target, num_classes=num_classes, class_reduction='none')
return accuracy, kc, iou, f1, precision, recall
def validation_step(self, batch, batch_idx):
self.model.eval()
tokens, intent_idx, entity_idx = batch
feature, intent_pred, entity_pred, entity_loss = self.forward(
tokens, entity_idx)
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
if entity_idx.sum().item() == 0 or torch.tensor(
entity_pred).sum().item() == 0:
entity_acc = get_token_accuracy(entity_idx.cpu(),
torch.tensor(entity_pred).cpu())[0]
else:
entity_acc = get_token_accuracy(
entity_idx.cpu(),
torch.tensor(entity_pred).cpu(),
ignore_index=self.dataset.pad_token_id)[0]
intent_loss = self.intent_loss_fn(
intent_pred,
intent_idx.long(),
) * self.intent_loss_weight
#intent_center_loss = self.intent_center_loss_fn(feature, intent_idx.long(),) * self.intent_loss_weight
entity_loss = -entity_loss * self.entity_loss_weight
return {
"val_intent_acc": torch.Tensor([intent_acc]),
"val_intent_f1": torch.Tensor([intent_f1]),
"val_entity_acc": torch.Tensor([entity_acc]),
"val_loss": intent_loss + entity_loss # + intent_center_loss,
}
def validation_epoch_end(self, outputs):
val_loss = sum([x["val_loss"] for x in outputs])
pred = torch.cat([x["pred"] for x in outputs])
true = torch.cat([x["true"] for x in outputs])
f_score = metrics.f1_score(pred, true)
accuracy = metrics.accuracy(pred, true)
out = {"val_loss": val_loss, "val_f_score": f_score, "val_accuracy": accuracy}
return {**out, "log": out}
def training_step(self, batch, batch_idx, optimizer_idx):
self.model.train()
tokens, intent_idx, entity_idx = batch
feature, intent_pred, entity_pred, entity_loss = self.forward(
tokens, entity_idx)
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
if entity_idx.sum().item() == 0 or torch.tensor(
entity_pred).sum().item() == 0:
entity_acc = get_token_accuracy(entity_idx.cpu(),
torch.tensor(entity_pred).cpu())[0]
else:
entity_acc = get_token_accuracy(
entity_idx.cpu(),
torch.tensor(entity_pred).cpu(),
ignore_index=self.dataset.pad_token_id)[0]
tensorboard_logs = {
"train/intent/acc": intent_acc,
"train/intent/f1": intent_f1,
"train/entity/acc": entity_acc,
}
if optimizer_idx == 0:
intent_loss = self.intent_loss_fn(
intent_pred,
intent_idx.long(),
) * self.intent_loss_weight
tensorboard_logs["train/intent/loss"] = intent_loss
return {
"loss": intent_loss,
"log": tensorboard_logs,
}
if optimizer_idx == 1:
entity_loss = -entity_loss * self.entity_loss_weight
tensorboard_logs["train/entity/loss"] = entity_loss
return {
"loss": entity_loss,
"log": tensorboard_logs,
}
if optimizer_idx == 2:
intent_center_loss = self.intent_center_loss_fn(
feature,
intent_idx.long(),
) * self.intent_center_loss_weight
tensorboard_logs["train/intent/center_loss"] = intent_center_loss
return {
"loss": intent_center_loss,
"log": tensorboard_logs,
}
def validation_epoch_end(self, outputs):
logits = torch.cat([o["logits"] for o in outputs], dim=0)
labels = torch.cat([o["labels"] for o in outputs], dim=0)
if self.hparams["multilabel"]:
self.log(
"val_auroc",
np.mean(
[
auroc(
torch.sigmoid(logits[:, i]),
labels[:, i],
pos_label=1,
)
.detach()
.cpu()
.item()
for i in range(logits.shape[1])
]
),
prog_bar=True,
)
else:
self.log(
"val_f1",
f1_score(
torch.argmax(logits, dim=1),
labels,
num_classes=self.hparams["output_size"],
class_reduction="macro",
)
.detach()
.cpu()
.item(),
prog_bar=True,
)
self.log(
"val_acc",
accuracy(torch.argmax(logits, dim=1), labels)
.detach()
.cpu()
.item(),
prog_bar=True,
)
self.log(
"val_loss",
self.loss(
logits,
labels.type(
logits.dtype if self.hparams["multilabel"] else labels.dtype
),
)
.detach()
.cpu()
.item(),
prog_bar=True,
)
def micro_F1_pytorch(y_true, y_scores):
""" Compute Accuracy
Args:
y_true (np.array): one hot encoded labels
y_scores (np.array): model prediction
Return:
auprc_macro (float): the accuracy
"""
f1 = plm.f1_score(torch.round(torch.flatten(y_scores)),
torch.flatten(y_true), 2)
return f1
def test_epoch_end(self, outputs):
test_loss = sum([x["test_loss"] for x in outputs])
pred = torch.cat([x["pred"] for x in outputs])
true = torch.cat([x["true"] for x in outputs])
f_score = metrics.f1_score(pred, true)
accuracy = metrics.accuracy(pred, true)
out = {
"test_loss": test_loss,
"test_f_score": f_score,
"test_accuracy": accuracy,
}
return {**out, "log": out}
def calculate_metrics(self, pred, target, prvs_metrics=None):
# output.cpu().detach().numpy()
metrics = {
'accuracy': pl_metrics.accuracy(torch.round(pred),
torch.round(target)),
'aucroc': pl_metrics.auroc(pred, target),
'f1_score': pl_metrics.f1_score(torch.round(pred),
torch.round(target))
}
if prvs_metrics is not None:
for key in metrics.keys():
metrics[key] += prvs_metrics[key]
metrics['counter'] = prvs_metrics['counter'] + 1
else:
metrics['counter'] = 1
return metrics
def training_step(self, batch, batch_idx, optimizer_idx):
self.model.train()
tokens, intent_idx, entity_idx = batch
intent_pred, entity_pred, entity_crf_loss = self.forward(
tokens, entity_idx)
if torch.isnan(tokens).sum().item() > 0:
assert ValueError('tokens error')
if torch.isnan(intent_idx).sum().item() > 0:
assert ValueError('intent_idx error')
if torch.isnan(entity_idx).sum().item() > 0:
assert ValueError('entity_idx error')
if torch.isnan(intent_pred).sum().item() > 0:
assert ValueError('intent_pred error')
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
entity_acc = get_token_accuracy(entity_idx.cpu(),
torch.tensor(entity_pred).cpu())[0]
tensorboard_logs = {
"train/intent/acc": intent_acc,
"train/intent/f1": intent_f1,
"train/entity/acc": entity_acc,
}
if optimizer_idx == 0:
intent_loss = self.intent_loss_fn(intent_pred, intent_idx.long())
tensorboard_logs["train/intent/loss"] = intent_loss
return {
"loss": intent_loss,
"log": tensorboard_logs,
}
if optimizer_idx == 1:
tensorboard_logs["train/entity/loss"] = entity_crf_loss
return {
"loss": entity_crf_loss,
"log": tensorboard_logs,
}
def validation_end(self, outputs):
scores = {}
for dataset_outputs in outputs:
# Assuming all val steps returned the same keys
keys = dataset_outputs[0].keys()
y_hat_key = list(filter(lambda x: 'y_hat' in x, keys))[0]
label_key = list(filter(lambda x: 'label' in x, keys))[0]
dataset_name = y_hat_key.split('-')[0]
y_hats = torch.cat(
[x[y_hat_key] for x in dataset_outputs if y_hat_key in x])
labels = torch.cat(
[x[label_key] for x in dataset_outputs if label_key in x])
scores[f'avg _{dataset_name}_f1'] = plm.f1_score(y_hats, labels)
avg_val_loss = torch.stack(
[x['val_loss'] for el in outputs for x in el]).mean()
scores['avg_val_loss'] = avg_val_loss
return {'log': scores}
def validation_epoch_end(self, outputs):
val_loss = sum([x["val_loss"] for x in outputs])
pred = torch.cat([x["pred"] for x in outputs])
true = torch.cat([x["true"] for x in outputs])
f_score = metrics.f1_score(pred, true)
accuracy = metrics.accuracy(pred, true)
# f_score = sk_metrics.f1_score(pred, true, average="macro")
# accuracy = sk_metrics.accuracy_score(pred, true)
out = {
"val_loss": val_loss,
"val_f_score": f_score,
"val_accuracy": accuracy,
"log": {
"val_loss": val_loss,
"val_f_score": f_score,
"val_accuracy": accuracy,
},
}
return out
def metrics(self, pred, target, num_classes=3, is_swnet=False):
if is_swnet:
pred[pred == 2] = 1
target[target == 2] = 1
num_classes = 2
confusion_m = mf.confusion_matrix(pred, target, num_classes=num_classes)
# acc
# try:
accuracy = confusion_m.diag().sum() / len(pred)
# except:
# print("pred:")
# print(pred)
# print("target:")
# print(target)
# print("confusion_m:")
# print("confusion_m")
# print("---")
# accuracy = 0
# kappa
# try:
p0 = accuracy
pc = 0
for i in range(confusion_m.shape[0]):
pc = pc + confusion_m[i].sum() * confusion_m[:, i].sum()
pc = pc / len(pred)**2
kc = (p0 - pc) / (1 - pc)
# if pc != 1:
# kc = (p0 - pc) / (1 - pc)
# else:
# kc = torch.tensor(1.0)
# kc.to(p0.device)
# except:
# kc = 0
f1 = mf.f1_score(pred, target, num_classes=num_classes, class_reduction='none')
precision = mf.precision(pred, target, num_classes=num_classes, class_reduction='none')
recall = mf.recall(pred, target, num_classes=num_classes, class_reduction='none')
return accuracy, kc, f1, precision, recall
def training_step(self, batch, batch_idx, optimizer_idx):
self.model.train()
tokens, intent_idx, entity_idx = batch
feature, = self.forward(tokens)
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
tensorboard_logs = {
"train/intent/acc": intent_acc,
"train/intent/f1": intent_f1,
"train/entity/acc": entity_acc,
}
if optimizer_idx == 0:
intent_loss = self.intent_loss_fn(intent_pred, intent_idx.long(),) * self.intent_loss_weight
tensorboard_logs["train/intent/loss"] = intent_loss
return {
"loss": intent_loss,
"log": tensorboard_logs,
}
if optimizer_idx == 1:
entity_loss = -entity_loss * self.entity_loss_weight
tensorboard_logs["train/entity/loss"] = entity_loss
return {
"loss": entity_loss,
"log": tensorboard_logs,
}
if optimizer_idx == 2:
intent_center_loss = self.intent_center_loss_fn(feature, intent_idx.long(),) * self.intent_center_loss_weight
tensorboard_logs["train/intent/center_loss"] = intent_center_loss
return {
"loss": intent_center_loss,
"log": tensorboard_logs,
}
def __metrics_per_batch(self, batch):
# 1. Forward pass:
x, y_true = batch
logits = self.forward(x)
# 2. Compute loss & performance metrics:
# class prediction: if binary (num_outputs == 1) then class label is 0 if logit < 0 else it's 1
# if multiclass then simply run argmax to find the index of the most confident class
y_hat = torch.argmax(logits, dim=1) if self.n_outputs > 1 else (
logits > 0.0).squeeze(1).long()
loss = self.loss(
logits, y_true if self.n_outputs > 1 else y_true.view(
(-1, 1)).type_as(x))
num_correct = torch.eq(y_hat, y_true.view(-1)).sum()
acc = num_correct.float() / self.batch_size
prec = plm.precision(y_hat, y_true, num_classes=self.n_classes)
rec = plm.recall(y_hat, y_true, num_classes=self.n_classes)
f1 = plm.f1_score(y_hat, y_true, num_classes=self.n_classes)
conf_matrix = plm.confusion_matrix(y_hat.long(), y_true.long())
acc1, acc5 = self.__accuracy(logits, y_true, topk=(1, 5))
return (y_true, y_hat, logits, loss, num_correct, acc, prec, rec, f1,
conf_matrix, acc1, acc5)
def validation_step(self, batch, batch_idx):
self.model.eval()
tokens, intent_idx, entity_idx = batch
intent_pred, entity_pred, entity_crf_loss = self.forward(
tokens, entity_idx)
intent_acc = get_accuracy(intent_pred.argmax(1), intent_idx)[0]
intent_f1 = f1_score(intent_pred.argmax(1), intent_idx)
entity_acc = get_token_accuracy(entity_idx.cpu(),
torch.tensor(entity_pred).cpu())[0]
intent_loss = self.intent_loss_fn(
intent_pred,
intent_idx.long(),
)
return {
"val_intent_acc": torch.Tensor([intent_acc]),
"val_intent_f1": torch.Tensor([intent_f1]),
"val_entity_acc": torch.Tensor([entity_acc]),
"val_loss": intent_loss + entity_crf_loss
}
train_f1 = 0
for X_batch, y_batch in train_loader:
X_batch, y_batch = X_batch.to(device), y_batch.to(device)
optimizer.zero_grad()
y_pred = model(X_batch)
loss = criterion(y_pred, y_batch)
train_loss += loss.item()
train_acc += accuracy(
y_batch, y_pred) # calculate accuracy (on this single batch)
train_recall += recall(y_batch, y_pred)
train_precision += train_precision(y_batch, y_pred)
train_f1 += f1_score(y_batch, y_pred)
loss.backward()
optimizer.step()
history['train_loss'].append(train_loss / len(train_loader))
history['train_acc'].append(train_acc / len(train_loader))
history['train_recall'].append(train_recall / len(train_loader))
history['train_precision'].append(train_precision / len(train_loader))
history['train_f1'].append(train_f1 / len(train_loader))
logger.info(f"EPOCH: {e} (training)")
logger.info(
f"{'':<10}Loss{'':<5} ----> {train_loss / len(train_loader):.5f}")
logger.info(
f"{'':<10}Accuracy{'':<1} ----> {train_acc / len(train_loader):.3f}"
def compute_metric(
metric: str,
predictions: Union[Sequence, torch.Tensor],
labels: Union[Sequence, torch.Tensor],
num_classes: int,
):
"""Compute metric given predictions and target labels
Args:
metric: name of metric
predictions: A sequence of predictions (rouge metrics) or a torch Tensor
(other metrics) containing predictions
labels: A sequence of labels (rouge metrics) or a torch Tensor (other metrics)
containing target labels
num_classes: number of classes
"""
if metric == "accuracy":
# Calculate the accuracy
if not isinstance(predictions, torch.Tensor):
predictions = torch.Tensor(predictions)
if not isinstance(labels, torch.Tensor):
labels = torch.Tensor(labels)
score = lightning_metrics.accuracy(
pred=predictions,
target=labels,
num_classes=num_classes,
).item()
elif metric == "f1":
# Calculate the f1
if not isinstance(predictions, torch.Tensor):
predictions = torch.Tensor(predictions)
if not isinstance(labels, torch.Tensor):
labels = torch.Tensor(labels)
score = lightning_metrics.f1_score(
pred=predictions,
target=labels,
num_classes=num_classes,
).item()
elif metric in ("Rouge-1", "Rouge-2", "Rouge-L"):
# Calculate rouge scores
if metric == "Rouge-1":
metric_id = "rouge1"
elif metric == "Rouge-2":
metric_id = "rouge2"
else:
metric_id = "rougeLsum"
scorer = rouge_scorer.RougeScorer([metric_id], use_stemmer=True)
# TODO Remove summarizaton-specific 'format_summary' call
# TODO Don't call scorer.score separately for each metric
score = statistics.mean(
scorer.score(format_summary(reference), format_summary(pred))
[metric].fmeasure for reference, pred in zip(labels, predictions))
elif metric == "class_dist":
# Calculate class distribution
if not isinstance(labels, torch.Tensor):
labels = torch.Tensor(labels)
score = (lightning_metrics.to_onehot(
tensor=labels,
num_classes=num_classes).double().mean(dim=0).tolist())
elif metric == "pred_dist":
# Calculate predicted class distribution
if not isinstance(predictions, torch.Tensor):
predictions = torch.Tensor(predictions)
score = (lightning_metrics.to_onehot(
tensor=predictions,
num_classes=num_classes).double().mean(dim=0).tolist())
else:
raise NotImplementedError
return score
