def test_classification_report(self):
classification_report_nemo = ClassificationReport(
num_classes=self.num_classes, label_ids=self.label_ids)
preds = torch.Tensor([0, 1, 1, 1, 2, 2, 0])
labels = torch.Tensor([1, 0, 0, 1, 2, 1, 0])
tp, fp, fn = classification_report_nemo(preds, labels)
def __convert_to_tensor(sklearn_metric):
return torch.Tensor([round(sklearn_metric * 100)])[0]
for mode in ['macro', 'micro', 'weighted']:
precision, recall, f1 = classification_report_nemo.get_precision_recall_f1(
tp, fn, fp, mode)
pr_sklearn, recall_sklearn, f1_sklearn, _ = precision_recall_fscore_support(
labels, preds, average=mode)
self.assertEqual(torch.round(precision),
__convert_to_tensor(pr_sklearn),
f'wrong precision for {mode}')
self.assertEqual(torch.round(recall),
__convert_to_tensor(recall_sklearn),
f'wrong recall for {mode}')
self.assertEqual(torch.round(f1), __convert_to_tensor(f1_sklearn),
f'wrong f1 for {mode}')
class TextClassificationModel(NLPModel, Exportable):
@property
def input_types(self) -> Optional[Dict[str, NeuralType]]:
return self.bert_model.input_types
@property
def output_types(self) -> Optional[Dict[str, NeuralType]]:
return self.classifier.output_types
def __init__(self, cfg: DictConfig, trainer: Trainer = None):
"""Initializes the BERTTextClassifier model."""
# shared params for dataset and data loaders
self.dataset_cfg = cfg.dataset
# tokenizer needs to get initialized before the super.__init__()
# as dataloaders and datasets need it to process the data
self._setup_tokenizer(cfg.tokenizer)
# init superclass
super().__init__(cfg=cfg, trainer=trainer)
self.bert_model = get_lm_model(
pretrained_model_name=cfg.language_model.pretrained_model_name,
config_file=cfg.language_model.config_file,
config_dict=cfg.language_model.config,
checkpoint_file=cfg.language_model.lm_checkpoint,
)
self.classifier = SequenceClassifier(
hidden_size=self.bert_model.config.hidden_size,
num_classes=cfg.dataset.num_classes,
num_layers=cfg.classifier_head.num_output_layers,
activation='relu',
log_softmax=False,
dropout=cfg.classifier_head.fc_dropout,
use_transformer_init=True,
idx_conditioned_on=0,
)
class_weights = None
if cfg.dataset.class_balancing == 'weighted_loss':
if cfg.train_ds.file_path:
class_weights = calc_class_weights(cfg.train_ds.file_path,
cfg.dataset.num_classes)
else:
logging.info(
'Class_balancing feature is enabled but no train file is given. Calculating the class weights is skipped.'
)
if class_weights:
# You may need to increase the number of epochs for convergence when using weighted_loss
self.loss = CrossEntropyLoss(weight=class_weights)
else:
self.loss = CrossEntropyLoss()
# setup to track metrics
self.classification_report = ClassificationReport(
cfg.dataset.num_classes)
def _setup_tokenizer(self, cfg: DictConfig):
tokenizer = get_tokenizer(
tokenizer_name=cfg.tokenizer_name,
vocab_file=self.register_artifact(
config_path='tokenizer.vocab_file', src=cfg.vocab_file),
special_tokens=OmegaConf.to_container(cfg.special_tokens)
if cfg.special_tokens else None,
tokenizer_model=self.register_artifact(
config_path='tokenizer.tokenizer_model',
src=cfg.tokenizer_model),
)
self.tokenizer = tokenizer
@typecheck()
def forward(self, input_ids, token_type_ids, attention_mask):
"""
No special modification required for Lightning, define it as you normally would
in the `nn.Module` in vanilla PyTorch.
"""
hidden_states = self.bert_model(input_ids=input_ids,
token_type_ids=token_type_ids,
attention_mask=attention_mask)
logits = self.classifier(hidden_states=hidden_states)
return logits
def training_step(self, batch, batch_idx):
"""
Lightning calls this inside the training loop with the data from the training dataloader
passed in as `batch`.
"""
# forward pass
input_ids, input_type_ids, input_mask, labels = batch
logits = self.forward(input_ids=input_ids,
token_type_ids=input_type_ids,
attention_mask=input_mask)
train_loss = self.loss(logits=logits, labels=labels)
tensorboard_logs = {
'train_loss': train_loss,
'lr': self._optimizer.param_groups[0]['lr']
}
return {'loss': train_loss, 'log': tensorboard_logs}
def validation_step(self, batch, batch_idx):
"""
Lightning calls this inside the validation loop with the data from the validation dataloader
passed in as `batch`.
"""
if self.testing:
prefix = 'test'
else:
prefix = 'val'
input_ids, input_type_ids, input_mask, labels = batch
logits = self.forward(input_ids=input_ids,
token_type_ids=input_type_ids,
attention_mask=input_mask)
val_loss = self.loss(logits=logits, labels=labels)
preds = torch.argmax(logits, axis=-1)
tp, fp, fn = self.classification_report(preds, labels)
tensorboard_logs = {
f'{prefix}_loss': val_loss,
f'{prefix}_tp': tp,
f'{prefix}_fn': fn,
f'{prefix}_fp': fp
}
return {f'{prefix}_loss': val_loss, 'log': tensorboard_logs}
def validation_epoch_end(self, outputs):
"""
Called at the end of validation to aggregate outputs.
:param outputs: list of individual outputs of each validation step.
"""
if not outputs:
return {}
if self.testing:
prefix = 'test'
else:
prefix = 'val'
avg_loss = torch.stack([x[f'{prefix}_loss'] for x in outputs]).mean()
# calculate metrics and log classification report
tp = torch.sum(
torch.stack([x['log'][f'{prefix}_tp'] for x in outputs]), 0)
fn = torch.sum(
torch.stack([x['log'][f'{prefix}_fn'] for x in outputs]), 0)
fp = torch.sum(
torch.stack([x['log'][f'{prefix}_fp'] for x in outputs]), 0)
precision, recall, f1 = self.classification_report.get_precision_recall_f1(
tp, fn, fp, mode='micro')
tensorboard_logs = {
f'{prefix}_loss': avg_loss,
f'{prefix}_precision': precision,
f'{prefix}_recall': recall,
f'{prefix}_f1': f1,
}
return {f'{prefix}_loss': avg_loss, 'log': tensorboard_logs}
def test_step(self, batch, batch_idx):
"""
Lightning calls this inside the test loop with the data from the test dataloader
passed in as `batch`.
"""
return self.validation_step(batch, batch_idx)
def test_epoch_end(self, outputs):
"""
Called at the end of test to aggregate outputs.
:param outputs: list of individual outputs of each test step.
"""
return self.validation_epoch_end(outputs)
def setup_training_data(self, train_data_config: Optional[DictConfig]):
if not train_data_config or not train_data_config.file_path:
logging.info(
f"Dataloader config or file_path for the train is missing, so no data loader for test is created!"
)
self._test_dl = None
return
self._train_dl = self._setup_dataloader_from_config(
cfg=train_data_config)
def setup_validation_data(self, val_data_config: Optional[DictConfig]):
if not val_data_config or not val_data_config.file_path:
logging.info(
f"Dataloader config or file_path for the validation is missing, so no data loader for test is created!"
)
self._test_dl = None
return
self._validation_dl = self._setup_dataloader_from_config(
cfg=val_data_config)
def setup_test_data(self, test_data_config: Optional[DictConfig]):
if not test_data_config or not test_data_config.file_path:
logging.info(
f"Dataloader config or file_path for the test is missing, so no data loader for test is created!"
)
self._test_dl = None
return
self._test_dl = self._setup_dataloader_from_config(
cfg=test_data_config)
def _setup_dataloader_from_config(
self, cfg: Dict) -> 'torch.utils.data.DataLoader':
input_file = cfg.file_path
if not os.path.exists(input_file):
raise FileNotFoundError(
f'{input_file} not found! The data should be be stored in TAB-separated files \n\
"validation_ds.file_path" and "train_ds.file_path" for train and evaluation respectively. \n\
Each line of the files contains text sequences, where words are separated with spaces. \n\
The label of the example is separated with TAB at the end of each line. \n\
Each line of the files should follow the format: \n\
[WORD][SPACE][WORD][SPACE][WORD][...][TAB][LABEL]')
dataset = TextClassificationDataset(
tokenizer=self.tokenizer,
input_file=input_file,
max_seq_length=self.dataset_cfg.max_seq_length,
num_samples=cfg.get("num_samples", -1),
shuffle=cfg.shuffle,
use_cache=self.dataset_cfg.use_cache,
)
return torch.utils.data.DataLoader(
dataset=dataset,
batch_size=cfg.batch_size,
shuffle=cfg.shuffle,
num_workers=cfg.get("num_workers", 0),
pin_memory=cfg.get("pin_memory", False),
drop_last=cfg.get("drop_last", False),
collate_fn=dataset.collate_fn,
)
@torch.no_grad()
def classifytext(self,
queries: List[str],
batch_size: int = 1,
max_seq_length: int = -1) -> List[int]:
"""
Get prediction for the queries
Args:
queries: text sequences
batch_size: batch size to use during inference
max_seq_length: sequences longer than max_seq_length will get truncated. default -1 disables truncation.
Returns:
all_preds: model predictions
"""
# store predictions for all queries in a single list
all_preds = []
mode = self.training
device = next(self.parameters()).device
try:
# Switch model to evaluation mode
self.eval()
logging_level = logging.get_verbosity()
logging.set_verbosity(logging.WARNING)
dataloader_cfg = {
"batch_size": batch_size,
"num_workers": 3,
"pin_memory": False
}
infer_datalayer = self._setup_infer_dataloader(
dataloader_cfg, queries, max_seq_length)
for i, batch in enumerate(infer_datalayer):
input_ids, input_type_ids, input_mask, subtokens_mask = batch
logits = self.forward(
input_ids=input_ids.to(device),
token_type_ids=input_type_ids.to(device),
attention_mask=input_mask.to(device),
)
preds = tensor2list(torch.argmax(logits, axis=-1))
all_preds.extend(preds)
finally:
# set mode back to its original value
self.train(mode=mode)
logging.set_verbosity(logging_level)
return all_preds
def _setup_infer_dataloader(
self,
cfg: Dict,
queries: List[str],
max_seq_length: int = -1) -> 'torch.utils.data.DataLoader':
"""
Setup function for a infer data loader.
Args:
cfg: config dictionary containing data loader params like batch_size, num_workers and pin_memory
queries: text
max_seq_length: maximum length of queries, default is -1 for no limit
Returns:
A pytorch DataLoader.
"""
dataset = TextClassificationDataset(tokenizer=self.tokenizer,
queries=queries,
max_seq_length=max_seq_length)
return torch.utils.data.DataLoader(
dataset=dataset,
batch_size=cfg["batch_size"],
shuffle=False,
num_workers=cfg.get("num_workers", 0),
pin_memory=cfg.get("pin_memory", False),
drop_last=False,
collate_fn=dataset.collate_fn,
)
@classmethod
def list_available_models(cls) -> Optional[Dict[str, str]]:
pass
@classmethod
def from_pretrained(cls, name: str):
pass
def _prepare_for_export(self):
return self.bert_model._prepare_for_export()
def export(
self,
output: str,
input_example=None,
output_example=None,
verbose=False,
export_params=True,
do_constant_folding=True,
keep_initializers_as_inputs=False,
onnx_opset_version: int = 12,
try_script: bool = False,
set_eval: bool = True,
check_trace: bool = True,
use_dynamic_axes: bool = True,
):
if input_example is not None or output_example is not None:
logging.warning(
"Passed input and output examples will be ignored and recomputed since"
" TextClassificationModel consists of two separate models with different"
" inputs and outputs.")
bert_model_onnx = self.bert_model.export(
'bert_' + output,
None, # computed by input_example()
None,
verbose,
export_params,
do_constant_folding,
keep_initializers_as_inputs,
onnx_opset_version,
try_script,
set_eval,
check_trace,
use_dynamic_axes,
)
classifier_onnx = self.classifier.export(
'classifier_' + output,
None, # computed by input_example()
None,
verbose,
export_params,
do_constant_folding,
keep_initializers_as_inputs,
onnx_opset_version,
try_script,
set_eval,
check_trace,
use_dynamic_axes,
)
output_model = attach_onnx_to_onnx(bert_model_onnx, classifier_onnx,
"CL")
onnx.save(output_model, output)
class IntentSlotClassificationModel(NLPModel):
@property
def input_types(self) -> Optional[Dict[str, NeuralType]]:
return self.bert_model.input_types
@property
def output_types(self) -> Optional[Dict[str, NeuralType]]:
return self.classifier.output_types
def __init__(self, cfg: DictConfig, trainer: Trainer = None):
""" Initializes BERT Joint Intent and Slot model.
"""
self.data_dir = cfg.data_dir
self.max_seq_length = cfg.language_model.max_seq_length
self.data_desc = IntentSlotDataDesc(
data_dir=cfg.data_dir,
modes=[cfg.train_ds.prefix, cfg.validation_ds.prefix])
self._setup_tokenizer(cfg.tokenizer)
# init superclass
super().__init__(cfg=cfg, trainer=trainer)
# initialize Bert model
self.bert_model = get_lm_model(
pretrained_model_name=cfg.language_model.pretrained_model_name,
config_file=cfg.language_model.config_file,
config_dict=OmegaConf.to_container(cfg.language_model.config)
if cfg.language_model.config else None,
checkpoint_file=cfg.language_model.lm_checkpoint,
)
self.classifier = SequenceTokenClassifier(
hidden_size=self.bert_model.config.hidden_size,
num_intents=self.data_desc.num_intents,
num_slots=self.data_desc.num_slots,
dropout=cfg.head.fc_dropout,
num_layers=cfg.head.num_output_layers,
log_softmax=False,
)
# define losses
if cfg.class_balancing == 'weighted_loss':
# You may need to increase the number of epochs for convergence when using weighted_loss
self.intent_loss = CrossEntropyLoss(
logits_ndim=2, weight=self.data_desc.intent_weights)
self.slot_loss = CrossEntropyLoss(
logits_ndim=3, weight=self.data_desc.slot_weights)
else:
self.intent_loss = CrossEntropyLoss(logits_ndim=2)
self.slot_loss = CrossEntropyLoss(logits_ndim=3)
self.total_loss = AggregatorLoss(
num_inputs=2,
weights=[cfg.intent_loss_weight, 1.0 - cfg.intent_loss_weight])
# setup to track metrics
self.intent_classification_report = ClassificationReport(
self.data_desc.num_intents, self.data_desc.intents_label_ids)
self.slot_classification_report = ClassificationReport(
self.data_desc.num_slots, self.data_desc.slots_label_ids)
# Optimizer setup needs to happen after all model weights are ready
self.setup_optimization(cfg.optim)
@typecheck()
def forward(self, input_ids, token_type_ids, attention_mask):
"""
No special modification required for Lightning, define it as you normally would
in the `nn.Module` in vanilla PyTorch.
"""
hidden_states = self.bert_model(input_ids=input_ids,
token_type_ids=token_type_ids,
attention_mask=attention_mask)
intent_logits, slot_logits = self.classifier(
hidden_states=hidden_states)
return intent_logits, slot_logits
def training_step(self, batch, batch_idx):
"""
Lightning calls this inside the training loop with the data from the training dataloader
passed in as `batch`.
"""
# forward pass
input_ids, input_type_ids, input_mask, loss_mask, subtokens_mask, intent_labels, slot_labels = batch
intent_logits, slot_logits = self(input_ids=input_ids,
token_type_ids=input_type_ids,
attention_mask=input_mask)
# calculate combined loss for intents and slots
intent_loss = self.intent_loss(logits=intent_logits,
labels=intent_labels)
slot_loss = self.slot_loss(logits=slot_logits,
labels=slot_labels,
loss_mask=loss_mask)
train_loss = self.total_loss(loss_1=intent_loss, loss_2=slot_loss)
tensorboard_logs = {
'train_loss': train_loss,
'lr': self._optimizer.param_groups[0]['lr']
}
return {'loss': train_loss, 'log': tensorboard_logs}
def validation_step(self, batch, batch_idx):
"""
Lightning calls this inside the validation loop with the data from the validation dataloader
passed in as `batch`.
"""
input_ids, input_type_ids, input_mask, loss_mask, subtokens_mask, intent_labels, slot_labels = batch
intent_logits, slot_logits = self(input_ids=input_ids,
token_type_ids=input_type_ids,
attention_mask=input_mask)
# calculate combined loss for intents and slots
intent_loss = self.intent_loss(logits=intent_logits,
labels=intent_labels)
slot_loss = self.slot_loss(logits=slot_logits,
labels=slot_labels,
loss_mask=loss_mask)
val_loss = self.total_loss(loss_1=intent_loss, loss_2=slot_loss)
# calculate accuracy metrics for intents and slot reporting
# intents
preds = torch.argmax(intent_logits, axis=-1)
intent_tp, intent_fp, intent_fn = self.intent_classification_report(
preds, intent_labels)
# slots
subtokens_mask = subtokens_mask > 0.5
preds = torch.argmax(slot_logits, axis=-1)[subtokens_mask]
slot_labels = slot_labels[subtokens_mask]
slot_tp, slot_fp, slot_fn = self.slot_classification_report(
preds, slot_labels)
tensorboard_logs = {
'val_loss': val_loss,
'intent_tp': intent_tp,
'intent_fn': intent_fn,
'intent_fp': intent_fp,
'slot_tp': slot_tp,
'slot_fn': slot_fn,
'slot_fp': slot_fp,
}
return {'val_loss': val_loss, 'log': tensorboard_logs}
def validation_epoch_end(self, outputs):
"""
Called at the end of validation to aggregate outputs.
:param outputs: list of individual outputs of each validation step.
"""
avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
# calculate metrics and log classification report (separately for intents and slots)
tp = torch.sum(torch.stack([x['log']['intent_tp'] for x in outputs]),
0)
fn = torch.sum(torch.stack([x['log']['intent_fn'] for x in outputs]),
0)
fp = torch.sum(torch.stack([x['log']['intent_fp'] for x in outputs]),
0)
intent_precision, intent_recall, intent_f1 = self.intent_classification_report.get_precision_recall_f1(
tp, fn, fp, mode='micro')
tp = torch.sum(torch.stack([x['log']['slot_tp'] for x in outputs]), 0)
fn = torch.sum(torch.stack([x['log']['slot_fn'] for x in outputs]), 0)
fp = torch.sum(torch.stack([x['log']['slot_fp'] for x in outputs]), 0)
slot_precision, slot_recall, slot_f1 = self.slot_classification_report.get_precision_recall_f1(
tp, fn, fp, mode='micro')
tensorboard_logs = {
'val_loss': avg_loss,
'intent_precision': intent_precision,
'intent_recall': intent_recall,
'intent_f1': intent_f1,
'slot_precision': slot_precision,
'slot_recall': slot_recall,
'slot_f1': slot_f1,
}
return {'val_loss': avg_loss, 'log': tensorboard_logs}
def test_step(self, batch, batch_idx):
"""
Lightning calls this inside the test loop with the data from the test dataloader
passed in as `batch`.
"""
return self.validation_step(batch, batch_idx)
def test_epoch_end(self, outputs):
"""
Called at the end of test to aggregate outputs.
:param outputs: list of individual outputs of each test step.
"""
return self.validation_epoch_end(outputs)
def _setup_tokenizer(self, cfg: DictConfig):
tokenizer = get_tokenizer(
tokenizer_name=cfg.tokenizer_name,
tokenizer_model=cfg.tokenizer_model,
special_tokens=OmegaConf.to_container(cfg.special_tokens)
if cfg.special_tokens else None,
vocab_file=cfg.vocab_file,
)
self.tokenizer = tokenizer
def setup_training_data(self, train_data_config: Optional[DictConfig]):
self._train_dl = self._setup_dataloader_from_config(
cfg=train_data_config)
def setup_validation_data(self, val_data_config: Optional[DictConfig]):
self._validation_dl = self._setup_dataloader_from_config(
cfg=val_data_config)
def setup_test_data(self, test_data_config: Optional[DictConfig]):
self._test_dl = self._setup_dataloader_from_config(
cfg=test_data_config)
def _setup_dataloader_from_config(self, cfg: DictConfig):
input_file = f'{self.data_dir}/{cfg.prefix}.tsv'
slot_file = f'{self.data_dir}/{cfg.prefix}_slots.tsv'
if not (os.path.exists(input_file) and os.path.exists(slot_file)):
raise FileNotFoundError(
f'{input_file} or {slot_file} not found. Please refer to the documentation for the right format \
of Intents and Slots files.')
dataset = IntentSlotClassificationDataset(
input_file=input_file,
slot_file=slot_file,
tokenizer=self.tokenizer,
max_seq_length=self.max_seq_length,
num_samples=cfg.num_samples,
pad_label=self.data_desc.pad_label,
ignore_extra_tokens=self._cfg.ignore_extra_tokens,
ignore_start_end=self._cfg.ignore_start_end,
)
return DataLoader(
dataset=dataset,
batch_size=cfg.batch_size,
shuffle=cfg.shuffle,
num_workers=cfg.num_workers,
pin_memory=cfg.pin_memory,
drop_last=cfg.drop_last,
collate_fn=dataset.collate_fn,
)
@classmethod
def list_available_models(cls) -> Optional[PretrainedModelInfo]:
"""
This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud.
Returns:
List of available pre-trained models.
"""
result = []
model = PretrainedModelInfo(
pretrained_model_name="Joint_Intent_Slot_Assistant",
location=
"https://api.ngc.nvidia.com/v2/models/nvidia/nemonlpmodels/versions/1.0.0a5/files/Joint_Intent_Slot_Assistant.nemo",
description=
"This models is trained on this https://github.com/xliuhw/NLU-Evaluation-Data dataset which includes 64 various intents and 55 slots. Final Intent accuracy is about 87%, Slot accuracy is about 89%.",
)
result.append(model)
return result
