def test_epoch(self, epoch_num: int):
"""Runs the test epoch for ``epoch_num``
Loads the best model that is saved during the training
and runs the test dataset.
Parameters
----------
epoch_num : int
zero based epoch number for which the test dataset is run
This is after the last training epoch.
"""
self.msg_printer.divider("Running on test batch")
self.load_model_from_file(self.save_dir.joinpath("best_model.pt"))
self.model.eval()
test_iter = iter(self.test_loader)
while True:
try:
iter_dict = next(test_iter)
iter_dict = move_to_device(obj=iter_dict, cuda_device=self.device)
with torch.no_grad():
model_forward_out = self.model(
iter_dict, is_training=False, is_validation=False, is_test=True
)
self.test_metric_calc.calc_metric(
iter_dict=iter_dict, model_forward_dict=model_forward_out
)
except StopIteration:
self.test_epoch_end(epoch_num)
break
def validation_epoch(self, epoch_num: int):
""" Runs one validation epoch on the validation dataset
Parameters
----------
epoch_num : int
0-based epoch number
"""
self.model.eval()
valid_iter = iter(self.validation_loader)
self.validation_loss_meter.reset()
self.validation_metric_calc.reset()
while True:
try:
iter_dict = next(valid_iter)
iter_dict = move_to_device(obj=iter_dict, cuda_device=self.device)
labels = iter_dict["label"]
batch_size = labels.size(0)
with torch.no_grad():
model_forward_out = self.model(
iter_dict, is_training=False, is_validation=True, is_test=False
)
loss = model_forward_out["loss"]
self.validation_loss_meter.add_loss(loss, batch_size)
self.validation_metric_calc.calc_metric(
iter_dict=iter_dict, model_forward_dict=model_forward_out
)
except StopIteration:
self.validation_epoch_end(epoch_num)
break
def infer_single_sentence(self, line: str) -> str:
""" Return the tagged string for a single sentence
Parameters
----------
line : str
A single sentence to be inferred
Returns
-------
str
Returns the tagged string for the line
"""
len_words = len(line.split())
iter_dict = self.dataset.get_iter_dict(line)
iter_dict = move_to_device(iter_dict, cuda_device=self.device)
iter_dict["tokens"] = iter_dict["tokens"].unsqueeze(0)
iter_dict["char_tokens"] = iter_dict["char_tokens"].unsqueeze(0)
model_output_dict = self.model_forward_on_iter_dict(
iter_dict=iter_dict)
_, predicted_tag_names = self.model_output_dict_to_prediction_indices_names(
model_output_dict=model_output_dict)
predicted_tag_names = predicted_tag_names[0].split()
len_pred_tag_names = len(predicted_tag_names)
infer_len = len_words if len_words < len_pred_tag_names else len_pred_tag_names
predicted_tag_names = predicted_tag_names[:infer_len]
predicted_tag_names = " ".join(predicted_tag_names)
return predicted_tag_names
def train_epoch(self, epoch_num: int):
"""
Run the training for one epoch
:param epoch_num: type: int
The current epoch number
"""
# refresh everything necessary before training begins
num_iterations = 0
train_iter = self.get_iter(self.train_loader)
self.train_loss_meter.reset()
self.train_metric_calc.reset()
self.model.train()
self.msg_printer.info("starting training epoch")
while True:
try:
# N*T, N * 1, N * 1
iter_dict = next(train_iter)
iter_dict = move_to_device(obj=iter_dict, cuda_device=self.device)
labels = iter_dict["label"]
batch_size = labels.size()[0]
model_forward_out = self.model(
iter_dict, is_training=True, is_validation=False, is_test=False
)
self.train_metric_calc.calc_metric(
iter_dict=iter_dict, model_forward_dict=model_forward_out
)
try:
self.optimizer.zero_grad()
loss = model_forward_out["loss"]
loss.backward()
torch.nn.utils.clip_grad_norm_(
self.model.parameters(), max_norm=self.gradient_norm_clip_value
)
self.optimizer.step()
self.train_loss_meter.add_loss(loss.item(), batch_size)
except KeyError:
self.msg_printer.fail(
"The model output dictionary does not have "
"a key called loss. Please check to have "
"loss in the model output"
)
num_iterations += 1
if (num_iterations + 1) % self.log_train_metrics_every == 0:
metrics = self.train_metric_calc.report_metrics()
print(metrics)
except StopIteration:
self.train_epoch_end(epoch_num)
break
def run_inference(self) -> Dict[str, Any]:
loader = DataLoader(
dataset=self.dataset, batch_size=self.batch_size, shuffle=False
)
output_analytics = {}
# contains the predicted class names for all the instances
pred_class_names = []
true_class_names = [] # contains the true class names for all the instances
sentences = [] # batch sentences in english
true_labels_indices = []
predicted_labels_indices = []
all_pred_probs = []
self.metrics_calculator.reset()
for iter_dict in loader:
iter_dict = move_to_device(obj=iter_dict, cuda_device=self.device)
batch_sentences = self.iter_dict_to_sentences(iter_dict)
model_output_dict = self.model_forward_on_iter_dict(iter_dict)
normalized_probs = model_output_dict["normalized_probs"]
self.metrics_calculator.calc_metric(
iter_dict=iter_dict, model_forward_dict=model_output_dict
)
true_label_ind, true_label_names = self.iter_dict_to_true_indices_names(
iter_dict=iter_dict
)
pred_label_indices, pred_label_names = self.model_output_dict_to_prediction_indices_names(
model_output_dict=model_output_dict
)
true_label_ind = torch.LongTensor(true_label_ind)
true_labels_indices.append(true_label_ind)
true_class_names.extend(true_label_names)
predicted_labels_indices.extend(pred_label_indices)
pred_class_names.extend(pred_label_names)
sentences.extend(batch_sentences)
all_pred_probs.append(normalized_probs)
# contains predicted probs for all the instances
all_pred_probs = torch.cat(all_pred_probs, dim=0)
true_labels_indices = torch.cat(true_labels_indices, dim=0).squeeze()
# torch.LongTensor N, 1
output_analytics["true_labels_indices"] = true_labels_indices
output_analytics["predicted_labels_indices"] = predicted_labels_indices
output_analytics["pred_class_names"] = pred_class_names
output_analytics["true_class_names"] = true_class_names
output_analytics["sentences"] = sentences
output_analytics["all_pred_probs"] = all_pred_probs
return output_analytics
def infer_single_sentence(self,
line: str) -> (List[str], List[str], List[str]):
""" Infers a single sentence and returns the labels
Parameters
----------
line : str
A single piece of text for inference
Returns
-------
(List[str], List[str], List[str])
Tagged sentences for task, process and material
"""
num_words = len(line.split())
iter_dict = self.dataset.get_iter_dict(line)
iter_dict = move_to_device(iter_dict, cuda_device=self.device)
iter_dict["tokens"] = iter_dict["tokens"].unsqueeze(0)
iter_dict["char_tokens"] = iter_dict["char_tokens"].unsqueeze(0)
model_output_dict = self.model_forward_on_iter_dict(
iter_dict=iter_dict)
(_, task_tag_names), (_, process_tag_names), (
_,
material_tag_names,
) = self.model_output_dict_to_prediction_indices_names(
model_output_dict=model_output_dict)
task_tag_names = task_tag_names[0].split()
process_tag_names = process_tag_names[0].split()
material_tag_names = material_tag_names[0].split()
len_tags = len(task_tag_names)
infer_len = num_words if num_words < len_tags else len_tags
task_tag_names = task_tag_names[:infer_len]
process_tag_names = process_tag_names[:infer_len]
material_tag_names = material_tag_names[:infer_len]
return task_tag_names, process_tag_names, material_tag_names
def run_inference(self) -> Dict[str, Any]:
loader = DataLoader(dataset=self.dataset,
batch_size=self.batch_size,
shuffle=False)
output_analytics = {}
sentences = [] # all the sentences that is seen till now
predicted_tag_indices = []
predicted_tag_names = [
] # all the tags that are predicted for the sentences
true_tag_indices = []
true_tag_names = []
for iter_dict in loader:
iter_dict = move_to_device(iter_dict, cuda_device=self.device)
model_output_dict = self.model_forward_on_iter_dict(
iter_dict=iter_dict)
self.metric_calc_on_iter_dict(iter_dict=iter_dict,
model_output_dict=model_output_dict)
batch_sentences = self.iter_dict_to_sentences(iter_dict=iter_dict)
predicted_tags, predicted_tag_strings = self.model_output_dict_to_prediction_indices_names(
model_output_dict=model_output_dict)
true_tags, true_labels_strings = self.iter_dict_to_true_indices_names(
iter_dict=iter_dict)
sentences.extend(batch_sentences)
predicted_tag_indices.extend(predicted_tags)
predicted_tag_names.extend(predicted_tag_strings)
true_tag_indices.extend(true_tags)
true_tag_names.extend(true_labels_strings)
output_analytics["true_tag_indices"] = true_tag_indices
output_analytics["predicted_tag_indices"] = predicted_tag_indices
output_analytics["true_tag_names"] = true_tag_names
output_analytics["predicted_tag_names"] = predicted_tag_names
output_analytics["sentences"] = sentences
return output_analytics