def validation_epoch_end(self, outputs):
if self.testing:
prefix = 'test'
else:
prefix = 'val'
avg_loss = torch.stack([x[f'{prefix}_loss'] for x in outputs]).mean()
unique_ids = torch.cat([x[f'{prefix}_tensors']['unique_ids'] for x in outputs])
start_logits = torch.cat([x[f'{prefix}_tensors']['start_logits'] for x in outputs])
end_logits = torch.cat([x[f'{prefix}_tensors']['end_logits'] for x in outputs])
all_unique_ids = []
all_start_logits = []
all_end_logits = []
if torch.distributed.is_initialized():
world_size = torch.distributed.get_world_size()
for ind in range(world_size):
all_unique_ids.append(torch.empty_like(unique_ids))
all_start_logits.append(torch.empty_like(start_logits))
all_end_logits.append(torch.empty_like(end_logits))
torch.distributed.all_gather(all_unique_ids, unique_ids)
torch.distributed.all_gather(all_start_logits, start_logits)
torch.distributed.all_gather(all_end_logits, end_logits)
else:
all_unique_ids.append(unique_ids)
all_start_logits.append(start_logits)
all_end_logits.append(end_logits)
exact_match, f1, all_predictions, all_nbest = -1, -1, [], []
if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0:
unique_ids = []
start_logits = []
end_logits = []
for u in all_unique_ids:
unique_ids.extend(tensor2list(u))
for u in all_start_logits:
start_logits.extend(tensor2list(u))
for u in all_end_logits:
end_logits.extend(tensor2list(u))
eval_dataset = self._test_dl.dataset if self.testing else self._validation_dl.dataset
exact_match, f1, all_predictions, all_nbest = eval_dataset.evaluate(
unique_ids=unique_ids,
start_logits=start_logits,
end_logits=end_logits,
n_best_size=self._cfg.dataset.n_best_size,
max_answer_length=self._cfg.dataset.max_answer_length,
version_2_with_negative=self._cfg.dataset.version_2_with_negative,
null_score_diff_threshold=self._cfg.dataset.null_score_diff_threshold,
do_lower_case=self._cfg.dataset.do_lower_case,
)
logging.info(f"{prefix} exact match {exact_match}")
logging.info(f"{prefix} f1 {f1}")
self.log(f'{prefix}_loss', avg_loss)
self.log(f'{prefix}_exact_match', exact_match)
self.log(f'{prefix}_f1', f1)
def _infer(self, sents: List[str]) -> List[List[int]]:
""" Main function for Inference
Args:
sents: A list of input sentences (lowercase spoken-domain words separated by space).
Returns:
all_preds: A list of tab-separated text records, same size as input list. Each record consists of 4 items:
- final output text
- input words
- tags predicted for input words
- tags after swap preprocessing
"""
# all input sentences go into one batch
dataloader_cfg = {
"batch_size": len(sents),
"num_workers": 3,
"pin_memory": False
}
infer_datalayer = self._setup_infer_dataloader(dataloader_cfg, sents)
batch = next(iter(infer_datalayer))
input_ids, input_mask, segment_ids = batch
tag_logits, semiotic_logits = self.forward(
input_ids=input_ids.to(self.device),
input_mask=input_mask.to(self.device),
segment_ids=segment_ids.to(self.device),
)
all_preds = []
for i, sent in enumerate(sents):
example = self.builder.build_bert_example(source=sent, infer=True)
tag_preds = tensor2list(torch.argmax(tag_logits[i], dim=-1))
semiotic_preds = tensor2list(
torch.argmax(semiotic_logits[i], dim=-1))
# this mask is required by get_token_labels
example.features["labels_mask"] = [
0
] + [1] * (len(semiotic_preds) - 2) + [0]
example.features["tag_labels"] = tag_preds
example.features["semiotic_labels"] = semiotic_preds
tags = [
self.id_2_tag[label_id]
for label_id in example.get_token_labels("tag_labels")
]
semiotic_labels = [
self.id_2_semiotic[label_id]
for label_id in example.get_token_labels("semiotic_labels")
]
prediction, inp_str, tag_str, tag_with_swap_str = example.editing_task.realize_output(
tags, semiotic_labels)
all_preds.append(prediction + "\t" + inp_str + "\t" + tag_str +
"\t" + tag_with_swap_str + "\t" +
" ".join(semiotic_labels))
return all_preds
def test_epoch_end(self, outputs):
unique_ids = tensor2list(
torch.cat([x['test_tensors']['unique_ids'] for x in outputs]))
logits = torch.cat([x['test_tensors']['logits'] for x in outputs])
s, e = logits.split(dim=-1, split_size=1)
start_logits = tensor2list(s.squeeze())
end_logits = tensor2list(e.squeeze())
(all_predictions, all_nbest,
scores_diff) = self.test_dataset.get_predictions(
unique_ids=unique_ids,
start_logits=start_logits,
end_logits=end_logits,
n_best_size=self._cfg.test_ds.n_best_size,
max_answer_length=self._cfg.test_ds.max_answer_length,
version_2_with_negative=self._cfg.dataset.version_2_with_negative,
null_score_diff_threshold=self._cfg.test_ds.
null_score_diff_threshold,
do_lower_case=self._cfg.dataset.do_lower_case,
)
if self._cfg.test_ds.output_nbest_file is not None:
with open(self._cfg.test_ds.output_nbest_file, "w") as writer:
writer.write(json.dumps(all_nbest, indent=4) + "\n")
if self._cfg.test_ds.output_prediction_file is not None:
with open(self._cfg.test_ds.output_prediction_file, "w") as writer:
writer.write(json.dumps(all_predictions, indent=4) + "\n")
return {}
def multi_validation_epoch_end(self, outputs, dataloader_idx: int = 0):
"""
Called at the end of validation to aggregate outputs.
outputs: list of individual outputs of each validation step.
"""
avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
preds = torch.cat([x['eval_tensors']['preds'] for x in outputs])
labels = torch.cat([x['eval_tensors']['labels'] for x in outputs])
all_preds = []
all_labels = []
if torch.distributed.is_initialized():
world_size = torch.distributed.get_world_size()
for ind in range(world_size):
all_preds.append(torch.empty_like(preds))
all_labels.append(torch.empty_like(labels))
torch.distributed.all_gather(all_preds, preds)
torch.distributed.all_gather(all_labels, labels)
else:
all_preds.append(preds)
all_labels.append(labels)
tensorboard_logs = {}
if not torch.distributed.is_initialized(
) or torch.distributed.get_rank() == 0:
preds = []
labels = []
for p in all_preds:
preds.extend(tensor2list(p))
for l in all_labels:
labels.extend(tensor2list(l))
tensorboard_logs = compute_metrics(self.task_name, np.array(preds),
np.array(labels))
val_name = self._validation_names[dataloader_idx].upper()
logging.info(f'{val_name} evaluation: {tensorboard_logs}')
# writing labels and predictions to a file in output_dir is specified in the config
output_dir = self._cfg.output_dir
if output_dir:
os.makedirs(output_dir, exist_ok=True)
filename = os.path.join(output_dir,
f'{self.task_name}_{val_name}.txt')
logging.info(f'Saving labels and predictions to {filename}')
with open(filename, 'w') as f:
f.write('labels\t' + list2str(labels) + '\n')
f.write('preds\t' + list2str(preds) + '\n')
tensorboard_logs['val_loss'] = avg_loss
for key in tensorboard_logs:
self.log(f'{key}', tensorboard_logs[key], prog_bar=True)
return {'val_loss': avg_loss, 'log': tensorboard_logs}
def calc_class_weights_from_dataloader(
dataloader: 'torch.utils.data.DataLoader', num_classes: int,
data_dir: str) -> List[float]:
"""
Calculate the weights of each class to be used for weighted loss. This is similar to the function calc_class_weights
in text_classification_dataset, but it gets the labels from a dataloader rather than from a file.
Args:
dataloader: the dataloader for the training set
num_classes: number of classes in the dataset
"""
labels = []
for batch in dataloader:
labels.extend(tensor2list(batch[-1]))
logging.info(f'Calculating label frequency stats...')
total_sents, sent_label_freq, max_id = get_label_stats(
labels, os.path.join(data_dir, 'sentence_stats.tsv'), verbose=False)
if max_id >= num_classes:
raise ValueError(
f'Found an invalid label! Labels should be from [0, num_classes-1].'
)
class_weights_dict = get_freq_weights(sent_label_freq)
logging.info(f'Total Sentence Pairs: {total_sents}')
logging.info(f'Class Frequencies: {sent_label_freq}')
logging.info(f'Class Weights: {class_weights_dict}')
class_weights = fill_class_weights(weights=class_weights_dict,
max_id=num_classes - 1)
return class_weights
def _infer(self, queries: List[str], batch_size: int = None) -> List[int]:
"""
Get prediction for the queries
Args:
queries: text sequences
batch_size: batch size to use during inference.
Returns:
all_preds: model predictions
"""
# store predictions for all queries in a single list
all_preds = []
mode = self.training
try:
device = 'cuda' if torch.cuda.is_available() else 'cpu'
# Switch model to evaluation mode
self.eval()
self.to(device)
infer_datalayer = self._setup_infer_dataloader(queries, batch_size)
for batch in infer_datalayer:
input_ids, input_type_ids, input_mask, subtokens_mask = batch
logits = self.forward(
input_ids=input_ids.to(device),
token_type_ids=input_type_ids.to(device),
attention_mask=input_mask.to(device),
)
subtokens_mask = subtokens_mask > 0.5
preds = tensor2list(torch.argmax(logits, axis=-1)[subtokens_mask])
all_preds.extend(preds)
finally:
# set mode back to its original value
self.train(mode=mode)
return all_preds
def predict_from_examples(self, queries: List[str],
test_ds) -> List[List[str]]:
"""
Get prediction for the queries (intent and slots)
Args:
queries: text sequences
test_ds: Dataset configuration section.
Returns:
predicted_intents, predicted_slots: model intent and slot predictions
"""
predicted_intents = []
predicted_slots = []
mode = self.training
device = 'cuda' if torch.cuda.is_available() else 'cpu'
# Switch model to evaluation mode
self.eval()
self.to(device)
# Dataset.
infer_datalayer = self._setup_infer_dataloader(queries, test_ds)
for batch in infer_datalayer:
input_ids, input_type_ids, input_mask, loss_mask, subtokens_mask = batch
intent_logits, slot_logits = self.forward(
input_ids=input_ids.to(device),
token_type_ids=input_type_ids.to(device),
attention_mask=input_mask.to(device),
)
# predict intents
intent_preds = tensor2list(torch.argmax(intent_logits, axis=-1))
predicted_intents += self.convert_intent_ids_to_intent_names(
intent_preds)
# predict slots
slot_preds = torch.argmax(slot_logits, axis=-1)
predicted_slots += self.mask_unused_subword_slots(
slot_preds, subtokens_mask)
# set mode back to its original value
self.train(mode=mode)
return predicted_intents, predicted_slots
def classifytext(self,
queries: List[str],
batch_size: int = 1,
max_seq_length: int = -1) -> List[int]:
"""
Get prediction for the queries
Args:
queries: text sequences
batch_size: batch size to use during inference
max_seq_length: sequences longer than max_seq_length will get truncated. default -1 disables truncation.
Returns:
all_preds: model predictions
"""
# store predictions for all queries in a single list
all_preds = []
mode = self.training
device = next(self.parameters()).device
try:
# Switch model to evaluation mode
self.eval()
logging_level = logging.get_verbosity()
logging.set_verbosity(logging.WARNING)
dataloader_cfg = {
"batch_size": batch_size,
"num_workers": 3,
"pin_memory": False
}
infer_datalayer = self._setup_infer_dataloader(
dataloader_cfg, queries, max_seq_length)
for i, batch in enumerate(infer_datalayer):
input_ids, input_type_ids, input_mask, subtokens_mask = batch
logits = self.forward(
input_ids=input_ids.to(device),
token_type_ids=input_type_ids.to(device),
attention_mask=input_mask.to(device),
)
preds = tensor2list(torch.argmax(logits, axis=-1))
all_preds.extend(preds)
finally:
# set mode back to its original value
self.train(mode=mode)
logging.set_verbosity(logging_level)
return all_preds
def main(args):
config_path = args.re_config
config = OmegaConf.load(config_path)
config.trainer.gpus = args.gpus
print(OmegaConf.to_yaml(config))
trainer = pl.Trainer(plugins=[NLPDDPPlugin()], **config.trainer)
exp_dir = exp_manager(trainer, config.get("exp_manager", None))
model = nemo_nlp.models.TextClassificationModel(cfg=config.model,
trainer=trainer)
trainer.fit(model)
model.save_to(config.model.nemo_path)
all_preds = []
mode = model.training
with torch.no_grad():
device = 'cuda' if torch.cuda.is_available() else 'cpu'
model.eval()
model.to(device)
# infer_datalayer = model._setup_infer_dataloader(queries, batch_size)
ts_dl = model._setup_dataloader_from_config(cfg=config.model.test_ds)
for i, batch in enumerate(ts_dl):
if i == 0:
print(batch)
input_ids, input_type_ids, input_mask, subtokens_mask = batch
logits = model.forward(
input_ids=input_ids.to(device),
token_type_ids=input_type_ids.to(device),
attention_mask=input_mask.to(device),
)
preds = tensor2list(torch.argmax(logits, axis=-1))
all_preds.extend(preds)
output_path = Path(args.pred_output)
output_path.mkdir(parents=True, exist_ok=True)
with open(output_path / "predict_labels.txt", "w") as f:
f.write("\n".join([str(e) for e in all_preds]))
def get_str_example_id(split: str, ids_to_service_names_dict: dict, example_id_num: torch.Tensor) -> str:
"""
Constructs string representation of example ID
Args:
split: evaluation data split
ids_to_service_names_dict: id to service name mapping
example_id_num: tensor example id
"""
def format_turn_id(ex_id_num):
dialog_id_1, dialog_id_2, turn_id, service_id, model_task_id, slot_intent_id, value_id = ex_id_num
return "{}-{}_{:05d}-{:02d}-{}-{}-{}-{}".format(
split,
dialog_id_1,
dialog_id_2,
turn_id,
ids_to_service_names_dict[service_id],
model_task_id,
slot_intent_id,
value_id,
)
return list(map(format_turn_id, tensor2list(example_id_num)))
def validation_epoch_end(self, outputs):
avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
unique_ids = torch.cat(
[x['eval_tensors']['unique_ids'] for x in outputs])
start_logits = torch.cat(
[x['eval_tensors']['start_logits'] for x in outputs])
end_logits = torch.cat(
[x['eval_tensors']['end_logits'] for x in outputs])
all_unique_ids = []
all_start_logits = []
all_end_logits = []
if torch.distributed.is_initialized():
world_size = torch.distributed.get_world_size()
for ind in range(world_size):
all_unique_ids.append(torch.empty_like(unique_ids))
all_start_logits.append(torch.empty_like(start_logits))
all_end_logits.append(torch.empty_like(end_logits))
torch.distributed.all_gather(all_unique_ids, unique_ids)
torch.distributed.all_gather(all_start_logits, start_logits)
torch.distributed.all_gather(all_end_logits, end_logits)
else:
all_unique_ids.append(unique_ids)
all_start_logits.append(start_logits)
all_end_logits.append(end_logits)
exact_match, f1, all_predictions, all_nbest = -1, -1, [], []
if not torch.distributed.is_initialized(
) or torch.distributed.get_rank() == 0:
unique_ids = []
start_logits = []
end_logits = []
for u in all_unique_ids:
unique_ids.extend(tensor2list(u))
for u in all_start_logits:
start_logits.extend(tensor2list(u))
for u in all_end_logits:
end_logits.extend(tensor2list(u))
exact_match, f1, all_predictions, all_nbest = self.validation_dataset.evaluate(
unique_ids=unique_ids,
start_logits=start_logits,
end_logits=end_logits,
n_best_size=self._cfg.validation_ds.n_best_size,
max_answer_length=self._cfg.validation_ds.max_answer_length,
version_2_with_negative=self._cfg.dataset.
version_2_with_negative,
null_score_diff_threshold=self._cfg.validation_ds.
null_score_diff_threshold,
do_lower_case=self._cfg.dataset.do_lower_case,
)
if self._cfg.validation_ds.output_nbest_file is not None:
with open(self._cfg.validation_ds.output_nbest_file,
"w") as writer:
writer.write(json.dumps(all_nbest, indent=4) + "\n")
if self._cfg.validation_ds.output_prediction_file is not None:
with open(self._cfg.validation_ds.output_prediction_file,
"w") as writer:
writer.write(json.dumps(all_predictions, indent=4) + "\n")
logging.info(f"exact match {exact_match}")
logging.info(f"f1 {f1}")
self.log('val_loss', avg_loss)
self.log('exact_match', exact_match)
self.log('f1', f1)
def inference(
self,
file: str,
batch_size: int = 1,
num_samples: int = -1,
output_nbest_file: Optional[str] = None,
output_prediction_file: Optional[str] = None,
):
"""
Get prediction for unlabeled inference data
Args:
file: inference data
batch_size: batch size to use during inference
num_samples: number of samples to use of inference data. Default: -1 if all data should be used.
output_nbest_file: optional output file for writing out nbest list
output_prediction_file: optional output file for writing out predictions
Returns:
model predictions, model nbest list
"""
# store predictions for all queries in a single list
all_predictions = []
all_nbest = []
mode = self.training
device = 'cuda' if torch.cuda.is_available() else 'cpu'
try:
# Switch model to evaluation mode
self.eval()
self.to(device)
logging_level = logging.get_verbosity()
logging.set_verbosity(logging.WARNING)
dataloader_cfg = {
"batch_size": batch_size,
"file": file,
"shuffle": False,
"num_samples": num_samples,
'num_workers': 2,
'pin_memory': False,
'drop_last': False,
}
dataloader_cfg = OmegaConf.create(dataloader_cfg)
infer_datalayer = self._setup_dataloader_from_config(
cfg=dataloader_cfg, mode=INFERENCE_MODE)
all_logits = []
all_unique_ids = []
for i, batch in enumerate(infer_datalayer):
input_ids, token_type_ids, attention_mask, unique_ids = batch
logits = self.forward(
input_ids=input_ids.to(device),
token_type_ids=token_type_ids.to(device),
attention_mask=attention_mask.to(device),
)
all_logits.append(logits)
all_unique_ids.append(unique_ids)
logits = torch.cat(all_logits)
unique_ids = tensor2list(torch.cat(all_unique_ids))
s, e = logits.split(dim=-1, split_size=1)
start_logits = tensor2list(s.squeeze(-1))
end_logits = tensor2list(e.squeeze(-1))
(all_predictions, all_nbest,
scores_diff) = infer_datalayer.dataset.get_predictions(
unique_ids=unique_ids,
start_logits=start_logits,
end_logits=end_logits,
n_best_size=self._cfg.dataset.n_best_size,
max_answer_length=self._cfg.dataset.max_answer_length,
version_2_with_negative=self._cfg.dataset.
version_2_with_negative,
null_score_diff_threshold=self._cfg.dataset.
null_score_diff_threshold,
do_lower_case=self._cfg.dataset.do_lower_case,
)
with open(file, 'r') as test_file_fp:
test_data = json.load(test_file_fp)["data"]
id_to_question_mapping = {}
for title in test_data:
for par in title["paragraphs"]:
for question in par["qas"]:
id_to_question_mapping[
question["id"]] = question["question"]
for question_id in all_predictions:
all_predictions[question_id] = (
id_to_question_mapping[question_id],
all_predictions[question_id])
if output_nbest_file is not None:
with open(output_nbest_file, "w") as writer:
writer.write(json.dumps(all_nbest, indent=4) + "\n")
if output_prediction_file is not None:
with open(output_prediction_file, "w") as writer:
writer.write(json.dumps(all_predictions, indent=4) + "\n")
finally:
# set mode back to its original value
self.train(mode=mode)
logging.set_verbosity(logging_level)
return all_predictions, all_nbest
def add_punctuation_capitalization(self, queries: List[str], batch_size: int = None) -> List[str]:
"""
Adds punctuation and capitalization to the queries. Use this method for debugging and prototyping.
Args:
queries: lower cased text without punctuation
batch_size: batch size to use during inference
Returns:
result: text with added capitalization and punctuation
"""
if queries is None or len(queries) == 0:
return []
if batch_size is None:
batch_size = len(queries)
logging.info(f'Using batch size {batch_size} for inference')
# We will store the output here
result = []
# Model's mode and device
mode = self.training
device = 'cuda' if torch.cuda.is_available() else 'cpu'
try:
# Switch model to evaluation mode
self.eval()
self = self.to(device)
infer_datalayer = self._setup_infer_dataloader(queries, batch_size)
# store predictions for all queries in a single list
all_punct_preds = []
all_capit_preds = []
for batch in infer_datalayer:
input_ids, input_type_ids, input_mask, subtokens_mask = batch
punct_logits, capit_logits = self.forward(
input_ids=input_ids.to(device),
token_type_ids=input_type_ids.to(device),
attention_mask=input_mask.to(device),
)
subtokens_mask = subtokens_mask > 0.5
punct_preds = tensor2list(torch.argmax(punct_logits, axis=-1)[subtokens_mask])
capit_preds = tensor2list(torch.argmax(capit_logits, axis=-1)[subtokens_mask])
all_punct_preds.extend(punct_preds)
all_capit_preds.extend(capit_preds)
queries = [q.strip().split() for q in queries]
queries_len = [len(q) for q in queries]
if sum(queries_len) != len(all_punct_preds) or sum(queries_len) != len(all_capit_preds):
raise ValueError('Pred and words must have the same length')
punct_ids_to_labels = {v: k for k, v in self._cfg.punct_label_ids.items()}
capit_ids_to_labels = {v: k for k, v in self._cfg.capit_label_ids.items()}
start_idx = 0
end_idx = 0
for query in queries:
end_idx += len(query)
# extract predictions for the current query from the list of all predictions
punct_preds = all_punct_preds[start_idx:end_idx]
capit_preds = all_capit_preds[start_idx:end_idx]
start_idx = end_idx
query_with_punct_and_capit = ''
for j, word in enumerate(query):
punct_label = punct_ids_to_labels[punct_preds[j]]
capit_label = capit_ids_to_labels[capit_preds[j]]
if capit_label != self._cfg.dataset.pad_label:
word = word.capitalize()
query_with_punct_and_capit += word
if punct_label != self._cfg.dataset.pad_label:
query_with_punct_and_capit += punct_label
query_with_punct_and_capit += ' '
result.append(query_with_punct_and_capit.strip())
finally:
# set mode back to its original value
self.train(mode=mode)
return result
def predict_from_examples(self,
queries: List[str],
batch_size: int = 32) -> List[List[str]]:
"""
Get prediction for the queries (intent and slots)
Args:
queries: text sequences
batch_size: batch size to use during inference
Returns:
predicted_intents, predicted_slots: model intent and slot predictions
"""
predicted_intents = []
predicted_slots = []
mode = self.training
try:
device = 'cuda' if torch.cuda.is_available() else 'cpu'
# Switch model to evaluation mode
self.eval()
self.to(device)
infer_datalayer = self._setup_infer_dataloader(queries, batch_size)
# load intent and slot labels from the dictionary files (user should have them in a data directory)
intent_labels, slot_labels = IntentSlotDataDesc.intent_slot_dicts(
self.data_dir)
for batch in infer_datalayer:
input_ids, input_type_ids, input_mask, loss_mask, subtokens_mask = batch
intent_logits, slot_logits = self.forward(
input_ids=input_ids.to(device),
token_type_ids=input_type_ids.to(device),
attention_mask=input_mask.to(device),
)
# predict intents and slots for these examples
# intents
intent_preds = tensor2list(torch.argmax(intent_logits,
axis=-1))
# convert numerical outputs to Intent and Slot labels from the dictionaries
for intent_num in intent_preds:
if intent_num < len(intent_labels):
predicted_intents.append(intent_labels[intent_num])
else:
# should not happen
predicted_intents.append("Unknown Intent")
# slots
slot_preds = torch.argmax(slot_logits, axis=-1)
for slot_preds_query, mask_query in zip(
slot_preds, subtokens_mask):
query_slots = ''
for slot, mask in zip(slot_preds_query, mask_query):
if mask == 1:
if slot < len(slot_labels):
query_slots += slot_labels[slot] + ' '
else:
query_slots += 'Unknown_slot '
predicted_slots.append(query_slots.strip())
finally:
# set mode back to its original value
self.train(mode=mode)
return predicted_intents, predicted_slots
def predict_from_examples(self, queries: List[str], test_ds) -> List[List[str]]:
"""
Get prediction for the queries (intent and slots)
Args:
queries: text sequences
test_ds: Dataset configuration section.
Returns:
predicted_intents, predicted_slots: model intent and slot predictions
"""
predicted_intents = []
predicted_slots = []
mode = self.training
try:
device = 'cuda' if torch.cuda.is_available() else 'cpu'
# Retrieve intent and slot vocabularies from configuration.
intent_labels = self.cfg.data_desc.intent_labels
slot_labels = self.cfg.data_desc.slot_labels
# Initialize tokenizer.
# if not hasattr(self, "tokenizer"):
# self._setup_tokenizer(self.cfg.tokenizer)
# Initialize modules.
# self._reconfigure_classifier()
# Switch model to evaluation mode
self.eval()
self.to(device)
# Dataset.
infer_datalayer = self._setup_infer_dataloader(queries, test_ds)
for batch in infer_datalayer:
input_ids, input_type_ids, input_mask, loss_mask, subtokens_mask = batch
intent_logits, slot_logits = self.forward(
input_ids=input_ids.to(device),
token_type_ids=input_type_ids.to(device),
attention_mask=input_mask.to(device),
)
# predict intents and slots for these examples
# intents
intent_preds = tensor2list(torch.argmax(intent_logits, axis=-1))
# convert numerical outputs to Intent and Slot labels from the dictionaries
for intent_num in intent_preds:
if intent_num < len(intent_labels):
predicted_intents.append(intent_labels[int(intent_num)])
else:
# should not happen
predicted_intents.append("Unknown Intent")
# slots
slot_preds = torch.argmax(slot_logits, axis=-1)
for slot_preds_query, mask_query in zip(slot_preds, subtokens_mask):
query_slots = ''
for slot, mask in zip(slot_preds_query, mask_query):
if mask == 1:
if slot < len(slot_labels):
query_slots += slot_labels[int(slot)] + ' '
else:
query_slots += 'Unknown_slot '
predicted_slots.append(query_slots.strip())
finally:
# set mode back to its original value
self.train(mode=mode)
return predicted_intents, predicted_slots
def predict_from_examples(
self, queries: List[str], test_ds: DictConfig, threshold: float = None
) -> Tuple[List[List[Tuple[str, float]]], List[str], List[List[int]]]:
"""
Get prediction for the queries (intent and slots)
Args:
queries: text sequences
test_ds: Dataset configuration section.
threshold: Threshold for rounding prediction logits
Returns:
predicted_intents: model intent predictions with their probabilities
Example: [[('flight', 0.84)], [('airfare', 0.54),
('flight', 0.73), ('meal', 0.24)]]
predicted_slots: model slot predictions
Example: ['O B-depart_date.month_name B-depart_date.day_number',
'O O B-flight_stop O O O']
predicted_vector: model intent predictions for each individual query. Binary values within each list
indicate whether a class is prediced for the given query (1 for True, 0 for False)
Example: [[1,0,0,0,0,0], [0,0,1,0,0,0]]
"""
predicted_intents = []
if threshold is None:
threshold = self.threshold
logging.info(f'Using threshold = {threshold}')
predicted_slots = []
predicted_vector = []
mode = self.training
try:
device = "cuda" if torch.cuda.is_available() else "cpu"
# Retrieve intent and slot vocabularies from configuration.
intent_labels = self.cfg.data_desc.intent_labels
slot_labels = self.cfg.data_desc.slot_labels
# Switch model to evaluation mode
self.eval()
self.to(device)
# Dataset.
infer_datalayer = self._setup_infer_dataloader(queries, test_ds)
for batch in infer_datalayer:
input_ids, input_type_ids, input_mask, loss_mask, subtokens_mask = batch
intent_logits, slot_logits = self.forward(
input_ids=input_ids.to(device),
token_type_ids=input_type_ids.to(device),
attention_mask=input_mask.to(device),
)
# predict intents and slots for these examples
# intents
intent_preds = tensor2list(torch.sigmoid(intent_logits))
# convert numerical outputs to Intent and Slot labels from the dictionaries
for intents in intent_preds:
intent_lst = []
temp_list = []
for intent_num, probability in enumerate(intents):
if probability >= threshold:
intent_lst.append((intent_labels[int(intent_num)], round(probability, 2)))
temp_list.append(1)
else:
temp_list.append(0)
predicted_vector.append(temp_list)
predicted_intents.append(intent_lst)
# slots
slot_preds = torch.argmax(slot_logits, axis=-1)
temp_slots_preds = []
for slot_preds_query, mask_query in zip(slot_preds, subtokens_mask):
temp_slots = ""
query_slots = ""
for slot, mask in zip(slot_preds_query, mask_query):
if mask == 1:
if slot < len(slot_labels):
query_slots += slot_labels[int(slot)] + " "
temp_slots += f"{slot} "
else:
query_slots += "Unknown_slot "
temp_slots += "0 "
predicted_slots.append(query_slots.strip())
temp_slots_preds.append(temp_slots)
finally:
# set mode back to its original value
self.train(mode=mode)
return predicted_intents, predicted_slots, predicted_vector
def add_punctuation_capitalization(
self, queries: List[str], batch_size: int = None, max_seq_length: int = 512
) -> List[str]:
"""
Adds punctuation and capitalization to the queries. Use this method for debugging and prototyping.
Args:
queries: lower cased text without punctuation
batch_size: batch size to use during inference
max_seq_length: maximum sequence length after tokenization
Returns:
result: text with added capitalization and punctuation
"""
if queries is None or len(queries) == 0:
return []
if batch_size is None:
batch_size = len(queries)
logging.info(f'Using batch size {batch_size} for inference')
# We will store the output here
result = []
# Model's mode and device
mode = self.training
device = 'cuda' if torch.cuda.is_available() else 'cpu'
try:
# Switch model to evaluation mode
self.eval()
self = self.to(device)
infer_datalayer = self._setup_infer_dataloader(queries, batch_size, max_seq_length)
# store predictions for all queries in a single list
all_punct_preds = []
all_capit_preds = []
for i, batch in tqdm(enumerate(infer_datalayer), total=len(infer_datalayer), desc="infer queries"):
input_ids, input_type_ids, input_mask, subtokens_mask = batch
punct_logits, capit_logits = self.forward(
input_ids=input_ids.to(device),
token_type_ids=input_type_ids.to(device),
attention_mask=input_mask.to(device),
)
subtokens_mask = subtokens_mask > 0.5
punct_preds = [
tensor2list(p_l[subtokens_mask[i]]) for i, p_l in enumerate(torch.argmax(punct_logits, axis=-1))
]
capit_preds = [
tensor2list(c_l[subtokens_mask[i]]) for i, c_l in enumerate(torch.argmax(capit_logits, axis=-1))
]
all_punct_preds.extend(punct_preds)
all_capit_preds.extend(capit_preds)
punct_ids_to_labels = {v: k for k, v in self._cfg.punct_label_ids.items()}
capit_ids_to_labels = {v: k for k, v in self._cfg.capit_label_ids.items()}
queries = [q.strip().split() for q in queries]
for i, query in enumerate(queries):
punct_preds = all_punct_preds[i]
capit_preds = all_capit_preds[i]
if len(query) != len(punct_preds):
logging.warning(
f'Max sequence length of query {query} is set to {max_seq_length}. Truncating the input.'
)
# removing the end of phrase punctuation of the truncated segment
punct_preds[-1] = 0
max_len = len(punct_preds)
query = query[:max_len]
query_with_punct_and_capit = ''
for j, word in enumerate(query):
punct_label = punct_ids_to_labels[punct_preds[j]]
capit_label = capit_ids_to_labels[capit_preds[j]]
if capit_label != self._cfg.dataset.pad_label:
word = word.capitalize()
query_with_punct_and_capit += word
if punct_label != self._cfg.dataset.pad_label:
query_with_punct_and_capit += punct_label
query_with_punct_and_capit += ' '
result.append(query_with_punct_and_capit.strip())
finally:
# set mode back to its original value
self.train(mode=mode)
return result
