def _validate(self, predictor, dataset):
# create a batch iterator for the given data
batch_iterator = batchIterator(self.voc,
dataset,
self.options['batch_size'],
shuffle=False)
# find out how many iterations we will need to cover the whole dataset
n_iters = len(dataset) // self.options['batch_size'] + int(
len(dataset) % self.options['batch_size'] > 0)
# containers for full prediction results so we can compute accuracy at the end
all_preds = []
all_labels = []
for iteration in range(1, n_iters + 1):
batch, batch_dialogs, true_batch_size = next(batch_iterator)
# Extract fields from batch
input_variable, dialog_lengths, utt_lengths, batch_indices, dialog_indices, \
batch_labels, batch_ids, target_variable, mask, max_target_len = batch
dialog_lengths_list = [len(x) for x in batch_dialogs]
# run the model
predictions, scores = self._evaluate_batch(
predictor, input_variable, dialog_lengths, dialog_lengths_list,
utt_lengths, batch_indices, dialog_indices, true_batch_size)
# aggregate results for computing accuracy at the end
all_preds += [p.item() for p in predictions]
all_labels += [l.item() for l in batch_labels]
print("Iteration: {}; Percent complete: {:.1f}%".format(
iteration, iteration / n_iters * 100))
# compute and return the accuracy
return (np.asarray(all_preds) == np.asarray(all_labels)).mean()
def _evaluate_dataset(self, predictor, dataset):
"""
Run a trained CRAFT model over an entire dataset in a batched fashion.
:param predictor: the trained CRAFT model to use, provided as a PyTorch Model instance.
:param dataset: the dataset to evaluate on, formatted as a list of (context, reply, id_of_reply) tuples.
:return: a DataFrame, indexed by utterance ID, of CRAFT scores for each utterance, and the corresponding binary prediction.
"""
# create a batch iterator for the given data
batch_iterator = batchIterator(self.voc,
dataset,
self.options['batch_size'],
shuffle=False)
# find out how many iterations we will need to cover the whole dataset
n_iters = len(dataset) // self.options['batch_size'] + int(
len(dataset) % self.options['batch_size'] > 0)
output_df = {
"id": [],
self.forecast_feat_name: [],
self.forecast_prob_feat_name: []
}
for iteration in range(1, n_iters + 1):
batch, batch_dialogs, true_batch_size = next(batch_iterator)
# Extract fields from batch
input_variable, dialog_lengths, utt_lengths, batch_indices, dialog_indices, \
labels, batch_ids, target_variable, mask, max_target_len = batch
dialog_lengths_list = [len(x) for x in batch_dialogs]
# run the model
predictions, scores = self._evaluate_batch(
predictor, input_variable, dialog_lengths, dialog_lengths_list,
utt_lengths, batch_indices, dialog_indices, true_batch_size)
# format the output as a dataframe (which we can later re-join with the corpus)
for i in range(true_batch_size):
utt_id = batch_ids[i]
pred = predictions[i].item()
score = scores[i].item()
output_df["id"].append(utt_id)
output_df[self.forecast_feat_name].append(pred)
output_df[self.forecast_prob_feat_name].append(score)
print("Iteration: {}; Percent complete: {:.1f}%".format(
iteration, iteration / n_iters * 100))
return pd.DataFrame(output_df).set_index("id")
def _train_iters(self, train_pairs, val_pairs, encoder, context_encoder,
attack_clf, encoder_optimizer, context_encoder_optimizer,
attack_clf_optimizer, embedding, n_iteration,
validate_every):
# create a batch iterator for training data
batch_iterator = batchIterator(self.voc, train_pairs,
self.options['batch_size'])
# Initializations
print('Initializing ...')
start_iteration = 1
print_loss = 0
# Training loop
print("Training...")
# keep track of best validation accuracy - only save when we have a model that beats the current best
best_acc = 0
for iteration in range(start_iteration, n_iteration + 1):
training_batch, training_dialogs, true_batch_size = next(
batch_iterator)
# Extract fields from batch
input_variable, dialog_lengths, utt_lengths, batch_indices, dialog_indices, \
labels, batch_ids, target_variable, mask, max_target_len = training_batch
dialog_lengths_list = [len(x) for x in training_dialogs]
# Run a training iteration with batch
loss = self._train_NN(
input_variable,
dialog_lengths,
dialog_lengths_list,
utt_lengths,
batch_indices,
dialog_indices,
labels, # input/output arguments
encoder,
context_encoder,
attack_clf, # network arguments
encoder_optimizer,
context_encoder_optimizer,
attack_clf_optimizer, # optimization arguments
true_batch_size,
self.options['clip']) # misc arguments
print_loss += loss
# Print progress
if iteration % self.options['print_every'] == 0:
print_loss_avg = print_loss / self.options['print_every']
print(
"Iteration: {}; Percent complete: {:.1f}%; Average loss: {:.4f}"
.format(iteration, iteration / n_iteration * 100,
print_loss_avg))
print_loss = 0
# Evaluate on validation set
if iteration % validate_every == 0:
print("Validating!")
# put the network components into evaluation mode
encoder.eval()
context_encoder.eval()
attack_clf.eval()
predictor = Predictor(encoder, context_encoder, attack_clf)
accuracy = self._validate(predictor, val_pairs)
print("Validation set accuracy: {:.2f}%".format(accuracy *
100))
# keep track of our best model so far
if accuracy > best_acc:
print(
"Validation accuracy better than current best; saving model..."
)
best_acc = accuracy
torch.save(
{
'iteration': iteration,
'en': encoder.state_dict(),
'ctx': context_encoder.state_dict(),
'atk_clf': attack_clf.state_dict(),
'en_opt': encoder_optimizer.state_dict(),
'ctx_opt': context_encoder_optimizer.state_dict(),
'atk_clf_opt': attack_clf_optimizer.state_dict(),
'loss': loss,
'voc_dict': self.voc.__dict__,
'embedding': embedding.state_dict()
}, self.options['trained_model_output_filepath'])
# put the network components back into training mode
encoder.train()
context_encoder.train()
attack_clf.train()