def load_model(filename, using_GPU):
RNNseq_model = RNNSequenceModel(num_classes=2, embedding_dim=300 + 1024, hidden_size=300, num_layers=1, bidir=True,
dropout1=0.5, dropout2=0, dropout3=0.1)
RNNseq_model.load_state_dict(torch.load(filename))
if using_GPU:
RNNseq_model.cuda()
return RNNseq_model
def load_model(filename, embedding_dim):
model = RNNSequenceModel(num_classes=2,
embedding_dim=embedding_dim,
hidden_size=hidden_size,
num_layers=1,
bidir=True,
dropout1=dropouts[0],
dropout2=dropouts[1],
dropout3=dropouts[2])
model.load_state_dict(torch.load(filename))
if using_GPU:
model.cuda()
return model
def train_model(train_dataloader,
val_dataloader,
embedding_dim,
num_epochs=20,
print_every=200,
prefix=dataset,
verbose=True):
model = RNNSequenceModel(num_classes=2,
embedding_dim=embedding_dim,
hidden_size=hidden_size,
num_layers=1,
bidir=True,
dropout1=dropouts[0],
dropout2=dropouts[1],
dropout3=dropouts[2])
model = model.cuda() if using_GPU else model
loss_weight = None
if class_weight:
loss_weight = torch.Tensor(class_weight)
loss_weight = loss_weight.cuda() if using_GPU else loss_weight
loss_criterion = nn.NLLLoss(weight=loss_weight,
reduction='sum' if class_weight else 'mean')
rnn_optimizer = optim.Adam(model.parameters(), lr=0.005)
optimal_score = [0, 0, 0, 0, 0] # Iteration, Acc, Precision, Recall, F1
optimal_state_dict = None
num_iter = 0 # Number of gradient updates
train_confusion_matrix = np.zeros(
(2,
2)) # Keep track of training performance - resets every 200 updates
for epoch in range(num_epochs):
# Slower learning rate
if epoch == num_epochs / 2:
rnn_optimizer = optim.Adam(model.parameters(), lr=0.001)
for (example_text, example_lengths, example_labels,
example_features) in train_dataloader:
example_text = Variable(example_text)
example_lengths = Variable(example_lengths)
example_labels = Variable(example_labels)
if using_GPU:
example_text = example_text.cuda()
example_lengths = example_lengths.cuda()
example_labels = example_labels.cuda()
# predicted shape: (batch_size, seq_len, 2)
predicted = model(example_text, example_lengths)
batch_loss = loss_criterion(predicted.view(-1, 2),
example_labels.view(-1))
rnn_optimizer.zero_grad()
batch_loss.backward()
rnn_optimizer.step()
num_iter += 1
# Get predictions, update confusion matrix
_, predicted_labels = torch.max(predicted.data, 2)
train_confusion_matrix = eval_util.update_confusion_matrix(
train_confusion_matrix, predicted_labels, example_labels.data)
# Calculate validation and training set loss and accuracy every 200 gradient updates
if num_iter % print_every == 0:
if verbose:
train_performance = eval_util.print_info(
train_confusion_matrix)
train_confusion_matrix = np.zeros((2, 2))
print(f"Iteration {num_iter}")
print(
f"Trn Performance: {train_performance}, Loss {batch_loss.item()}"
)
if val_dataloader is not None:
avg_eval_loss, performance = eval_util.evaluate(
val_dataloader, model, loss_criterion, using_GPU)
if performance[-1] > optimal_score[-1]:
optimal_score = performance
optimal_state_dict = model.state_dict()
if verbose:
print(
f"Val Performance: {performance}, Loss {avg_eval_loss}"
)
filename = f"models/{prefix}_iter_{num_iter}.pt"
torch.save(model.state_dict(), filename)
model.load_state_dict(optimal_state_dict)
return model, optimal_score
idx2pos,
val_dataloader_vua,
loss_criterion,
using_GPU,
RNNseq_model,
Transformer_model,
)
val_end_f1 = 2 * precision * recall / (precision + recall)
if val_end_f1 > bestf1:
bestf1 = val_end_f1
best_model_weights[0] = copy.deepcopy(RNNseq_model.state_dict())
best_model_weights[1] = copy.deepcopy(Transformer_model.state_dict())
best_optimizer_dict[0] = copy.deepcopy(rnn_optimizer.state_dict())
best_optimizer_dict[1] = copy.deepcopy(trans_optimizer.state_dict())
RNNseq_model.load_state_dict(best_model_weights[0])
Transformer_model.load_state_dict(best_model_weights[1])
rnn_optimizer.load_state_dict(best_optimizer_dict[0])
trans_optimizer.load_state_dict(best_optimizer_dict[1])
torch.save(
{
'epoch': num_epochs,
'model_state_dict': best_model_weights,
'optimizer_state_dict': best_optimizer_dict,
}, './models/vua/model.tar')
"""
3.3
plot the training process: losses for validation and training dataset
"""
plt.figure(0)
