def train_model(model,
name,
train_dataloader,
validation_dataloader,
filepath='',
lr=2e-5,
EPOCHS=7,
BATCH_SIZE=1,
weight_decay=0.9,
eps=1e-7):
training_stats = []
torch.cuda.empty_cache()
model = model.to(DEVICE)
optimizer = AdamW(model.parameters(),
lr=lr,
weight_decay=weight_decay,
eps=eps)
loss_func = torch.nn.NLLLoss()
training_loss_history = []
val_loss_history = []
for epoch_num in range(EPOCHS):
t0 = time.time()
model.train()
total_train_loss = 0
for step_num, batch_data in enumerate(train_dataloader):
input_ids, attention_masks, anger, fear, joy, sadness, vec, intensity = tuple(
t for t in batch_data)
input_ids = input_ids.to(DEVICE)
attention_masks = attention_masks.to(DEVICE)
anger = anger.to(DEVICE)
fear = fear.to(DEVICE)
joy = joy.to(DEVICE)
sadness = sadness.to(DEVICE)
intensity = intensity.to(DEVICE)
vec = vec.to(DEVICE)
model.zero_grad()
probas = model(input_ids, attention_masks)
loss = loss_func(probas, intensity)
total_train_loss += loss.item()
loss.backward()
torch.torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
# scheduler.step()
print('Epoch: ', epoch_num + 1)
print("\r" + "{0}/{1} loss: {2} ".format(
step_num,
len(train_dataloader) / BATCH_SIZE, total_train_loss /
(step_num + 1)))
avg_train_loss = total_train_loss / len(train_dataloader)
dropout = model.dropout.p
parameter_tuned = '_lr_' + str(lr) + '_dropout_' + str(
dropout) + '_weight_decay_' + str(weight_decay) + '_eps_' + str(
eps)
model_save_name = filepath + name + '_epoch_' + str(
epoch_num) + parameter_tuned + '.pt'
torch.save(model.state_dict(), model_save_name)
training_time = format_time(time.time() - t0)
model.eval()
total_pearson = 0
total_pearson_some = 0
total_kappa = 0
total_kappa_some = 0
total_eval_loss = 0
for batch_data in validation_dataloader:
input_ids, attention_masks, anger, fear, joy, sadness, vec, intensity = tuple(
t for t in batch_data)
input_ids = input_ids.to(DEVICE)
attention_masks = attention_masks.to(DEVICE)
anger = anger.to(DEVICE)
fear = fear.to(DEVICE)
joy = joy.to(DEVICE)
sadness = sadness.to(DEVICE)
intensity = intensity.to(DEVICE)
vec = vec.to(DEVICE)
with torch.no_grad():
probas = model(input_ids, attention_masks)
output = torch.max(probas, 1)[1]
loss = loss_func(probas, intensity)
# Accumulate the validation loss.
total_eval_loss += loss.item()
output = output.detach().cpu()
intensity = intensity.to('cpu')
# Calculate the accuracy for this batch of test sentences, and
# accumulate it over all batches.
pear, pear_some, kappa, kappa_some = evaluate_PerEmotion(
intensity, output)
total_pearson += pear
total_pearson_some += pear_some
total_kappa += kappa
total_kappa_some += kappa_some
# Report the final accuracy for this validation run.
avg_pearson = total_pearson / len(validation_dataloader)
avg_some_pearson = total_pearson_some / len(validation_dataloader)
avg_kappa = total_kappa / len(validation_dataloader)
avg_some_kappa = total_kappa_some / len(validation_dataloader)
# Calculate the average loss over all of the batches.
avg_val_loss = total_eval_loss / len(validation_dataloader)
val_time = format_time(time.time() - t0)
training_loss_history.append(avg_train_loss)
val_loss_history.append(avg_val_loss)
# Record all statistics from this epoch.
training_stats.append({
'epoch': epoch_num + 1,
'Training Loss': avg_train_loss,
'Valid. Loss': avg_val_loss,
'Pearson': avg_pearson,
'Pearson Some': avg_some_pearson,
'Kappa': avg_kappa,
'Kappa Some': avg_some_kappa,
'Learning Rate': lr,
'Weight Decay': weight_decay,
'Dropout': dropout,
'Epsilon': eps,
'Training Time': training_time,
'Validation Time': val_time
})
return training_stats, training_loss_history, val_loss_history, parameter_tuned
def train(weight,
model_num,
model,
train_dataloader,
validation_dataloader,
filepath='',
lr=2e-5,
EPOCHS=10,
BATCH_SIZE=1):
total_t0 = time.time()
training_stats = []
model = model.to(DEVICE)
optimizer = AdamW(model.parameters(), lr=lr)
weight = weight.to(DEVICE)
loss_func = nn.NLLLoss(weight)
loss_real = nn.NLLLoss()
softmax = nn.LogSoftmax(dim=1)
for epoch_num in range(EPOCHS):
t0 = time.time()
model.train()
total_train_loss = 0
for step_num, batch_data in enumerate(train_dataloader):
input_ids, attention_masks, anger, fear, joy, sadness, vec, intensity = tuple(
t for t in batch_data)
##ordinal
o1 = torch.tensor((intensity.numpy() > 0).astype(int))
o2 = torch.tensor((intensity.numpy() > 1).astype(int))
o3 = torch.tensor((intensity.numpy() > 2).astype(int))
if model_num == 1:
o = o1
if model_num == 2:
o = o2
if model_num == 3:
o = o3
###
input_ids = input_ids.to(DEVICE)
attention_masks = attention_masks.to(DEVICE)
anger = anger.to(DEVICE)
fear = fear.to(DEVICE)
joy = joy.to(DEVICE)
sadness = sadness.to(DEVICE)
intensity = intensity.to(DEVICE)
vec = vec.to(DEVICE)
o = o.to(DEVICE)
model.zero_grad()
probas = model(input_ids, attention_masks)
loss = loss_func(probas, o)
total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
# scheduler.step()
print('Epoch: ', epoch_num + 1)
print("\r" + "{0}/{1} loss: {2} ".format(
step_num,
len(train_dataloader) / BATCH_SIZE, total_train_loss /
(step_num + 1)))
avg_train_loss = total_train_loss / len(train_dataloader)
#model_save_name = filepath + '_epoch_' + str(epoch_num) + '_lr_' + str(lr) + '_' + str(model_num) + '.pt'
#torch.save(model.state_dict(), model_save_name)
training_time = format_time(time.time() - t0)
model.eval()
total_pearson = 0
total_kappa = 0
total_eval_loss_r = 0
total_eval_loss_w = 0
for batch_data in validation_dataloader:
input_ids, attention_masks, anger, fear, joy, sadness, vec, intensity = tuple(
t for t in batch_data)
##ordinal
o1 = torch.tensor((intensity.numpy() > 0).astype(int))
o2 = torch.tensor((intensity.numpy() > 1).astype(int))
o3 = torch.tensor((intensity.numpy() > 2).astype(int))
if model_num == 1:
o = o1
elif model_num == 2:
o = o2
else:
o = o3
###
input_ids = input_ids.to(DEVICE)
attention_masks = attention_masks.to(DEVICE)
anger = anger.to(DEVICE)
fear = fear.to(DEVICE)
joy = joy.to(DEVICE)
sadness = sadness.to(DEVICE)
intensity = intensity.to(DEVICE)
vec = vec.to(DEVICE)
o = o.to(DEVICE)
with torch.no_grad():
probas = model(input_ids, attention_masks)
output = torch.max(probas, 1)[1]
lossr = loss_real(probas, o)
lossw = loss_func(probas, o)
# Accumulate the validation loss.
total_eval_loss_r += lossr.item()
total_eval_loss_w += lossw.item()
output = output.detach().cpu()
o = o.to('cpu')
# Calculate the accuracy for this batch of test sentences, and
# accumulate it over all batches.
pear, _, kappa, _ = evaluate_PerEmotion(o, output)
total_pearson += pear
total_kappa += kappa
# Report the final accuracy for this validation run.
avg_pearson = total_pearson / len(validation_dataloader)
avg_kappa = total_kappa / len(validation_dataloader)
# Calculate the average loss over all of the batches.
avg_val_loss_r = total_eval_loss_r / len(validation_dataloader)
avg_val_loss_w = total_eval_loss_w / len(validation_dataloader)
val_time = format_time(time.time() - t0)
# Record all statistics from this epoch.
training_stats.append({
'epoch': epoch_num + 1,
'Training Loss on 1 ordinal': avg_train_loss,
'Valid. Loss on 1 ordinal, real': avg_val_loss_r,
'Valid. Loss on 1 ordinal, weighted': avg_val_loss_w,
'Pearson on 1 ordinal': avg_pearson,
'Kappa on 1 ordinal': avg_kappa,
'Learning Rate': lr,
'Training Time': training_time,
'Validation Time': val_time
})
print(training_stats)
return training_stats, model
def train(model,
train_dataloader,
validation_dataloader,
filepath='',
lr=2e-5,
EPOCHS=10,
BATCH_SIZE=1,
dropout=0.1,
hidden_size=20):
training_stats = []
model = model.to(DEVICE)
optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, model.parameters()), lr)
loss_func = torch.nn.NLLLoss()
for epoch_num in range(EPOCHS):
t0 = time.time()
model.train()
total_train_loss = 0
for _, batch_data in enumerate(train_dataloader):
tweet = batch_data.Clean_Tweet
intensity = batch_data.Intensity.to(DEVICE)
probas = model(tweet[0].to(DEVICE), tweet[1].to(DEVICE))
loss = loss_func(probas, intensity)
total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
avg_train_loss = total_train_loss / len(train_dataloader)
training_time = str(
datetime.timedelta(seconds=int(round(((time.time() - t0))))))
model.eval()
outputs = []
intensities = []
total_eval_loss = 0
for batch_data in validation_dataloader:
tweet = batch_data.Clean_Tweet
intensity = batch_data.Intensity.to(DEVICE)
with torch.no_grad():
probas = model(tweet[0].to(DEVICE), tweet[1].to(DEVICE))
output = torch.max(probas, 1)[1]
loss = loss_func(probas, intensity)
total_eval_loss += loss.item()
output = output.detach().cpu()
intensity = intensity.to('cpu')
outputs.extend(list(output.numpy()))
intensities.extend(list(intensity.numpy()))
outputs = torch.Tensor(outputs)
intensities = torch.Tensor(intensities)
pear, pear_some, kappa, kappa_some = evaluate_PerEmotion(
intensities, outputs)
avg_val_loss = total_eval_loss / len(validation_dataloader)
val_time = str(
datetime.timedelta(seconds=int(round(((time.time() - t0))))))
training_stats.append({
'epoch': epoch_num + 1,
'Training Loss': avg_train_loss,
'Valid. Loss': avg_val_loss,
'Pearson': pear,
'Pearson Some': pear_some,
'Kappa': kappa,
'Kappa Some': kappa_some,
'Learning Rate': lr,
'Dropout': dropout,
'Hidden Size': hidden_size,
'Training Time': training_time,
'Validation Time': val_time
})
return training_stats
def train_model(model, train_dataloader, validation_dataloader, filepath='', lr=2e-5, EPOCHS=5, BATCH_SIZE=1, weight_decay=0.9, drop_out=0.1):
training_stats = []
torch.cuda.empty_cache()
model = model.to(DEVICE)
optimizer = AdamW(model.parameters(), lr = lr, weight_decay=weight_decay)
loss_func = torch.nn.NLLLoss()
for epoch_num in range(EPOCHS):
t0 = time.time()
model.train()
total_train_loss = 0
for step_num, batch_data in enumerate(train_dataloader):
input_ids, attention_masks, anger, fear, joy, sadness, vec, intensity = tuple(t for t in batch_data)
input_ids = input_ids.to(DEVICE)
attention_masks = attention_masks.to(DEVICE)
anger = anger.to(DEVICE)
fear = fear.to(DEVICE)
joy = joy.to(DEVICE)
sadness = sadness.to(DEVICE)
intensity = intensity.to(DEVICE)
vec = vec.to(DEVICE)
model.zero_grad()
probas = model(input_ids, vec, attention_masks)
loss = loss_func(probas, intensity)
total_train_loss += loss.item()
loss.backward()
torch.torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
print('Epoch: ', epoch_num + 1)
print("\r" + "{0}/{1} loss: {2} ".format(step_num, len(train_dataloader) / BATCH_SIZE, total_train_loss / (step_num + 1)))
avg_train_loss = total_train_loss / len(train_dataloader)
training_time = str(datetime.timedelta(seconds=int(round(((time.time() - t0))))))
model.eval()
total_eval_loss = 0
anger_emotion_intensity = []
anger_emotion_output = []
fear_emotion_intensity = []
fear_emotion_output = []
sadness_emotion_intensity = []
sadness_emotion_output = []
joy_emotion_intensity = []
joy_emotion_output = []
for batch_data in validation_dataloader:
input_ids, attention_masks, anger, fear, joy, sadness, vec, intensity = tuple(t for t in batch_data)
input_ids = input_ids.to(DEVICE)
attention_masks = attention_masks.to(DEVICE)
anger = anger.to(DEVICE)
fear = fear.to(DEVICE)
joy = joy.to(DEVICE)
sadness = sadness.to(DEVICE)
intensity = intensity.to(DEVICE)
vec = vec.to(DEVICE)
with torch.no_grad():
probas = model(input_ids, vec, attention_masks)
output = torch.max(probas, 1)[1]
loss = loss_func(probas, intensity)
total_eval_loss += loss.item()
output = output.detach().cpu()
intensity = intensity.to('cpu')
for i, a in enumerate(anger.to('cpu')):
if int(a) == 1:
anger_emotion_intensity.append(intensity[i])
anger_emotion_output.append(output[i])
for i, a in enumerate(fear.to('cpu')):
if int(a) == 1:
fear_emotion_intensity.append(intensity[i])
fear_emotion_output.append(output[i])
for i, a in enumerate(sadness.to('cpu')):
if int(a) == 1:
sadness_emotion_intensity.append(intensity[i])
sadness_emotion_output.append(output[i])
for i, a in enumerate(joy.to('cpu')):
if int(a) == 1:
joy_emotion_intensity.append(intensity[i])
joy_emotion_output.append(output[i])
anger_emotion_intensity = torch.Tensor(anger_emotion_intensity)
anger_emotion_output = torch.Tensor(anger_emotion_output)
fear_emotion_intensity = torch.Tensor(fear_emotion_intensity)
fear_emotion_output = torch.Tensor(fear_emotion_output)
sadness_emotion_intensity = torch.Tensor(sadness_emotion_intensity)
sadness_emotion_output = torch.Tensor(sadness_emotion_output)
joy_emotion_intensity = torch.Tensor(joy_emotion_intensity)
joy_emotion_output = torch.Tensor(joy_emotion_output)
pear_fear, pear_some_fear, kappa_fear, kappa_some_fear = evaluate_PerEmotion (fear_emotion_intensity, fear_emotion_output)
pear_joy, pear_some_joy, kappa_joy, kappa_some_joy = evaluate_PerEmotion (joy_emotion_intensity, joy_emotion_output)
pear_anger, pear_some_anger, kappa_anger, kappa_some_anger = evaluate_PerEmotion (anger_emotion_intensity, anger_emotion_output)
pear_sadness, pear_some_sadness, kappa_sadness, kappa_some_sadness = evaluate_PerEmotion (sadness_emotion_intensity, sadness_emotion_output)
pears = np.mean((pear_fear, pear_joy, pear_sadness, pear_anger))
pears_some = np.mean((pear_some_fear, pear_some_joy, pear_some_sadness, pear_some_anger))
kappa = np.mean((kappa_fear, kappa_joy, kappa_sadness, kappa_anger))
kappa_some = np.mean((kappa_some_fear, kappa_some_joy, kappa_some_sadness, kappa_some_anger))
# Calculate the average loss over all of the batches.
avg_val_loss = total_eval_loss / len(validation_dataloader)
val_time = str(datetime.timedelta(seconds=int(round(((time.time() - t0))))))
# Record all statistics from this epoch.
training_stats.append({
'epoch': epoch_num + 1,
'Training Loss': avg_train_loss,
'Valid. Loss': avg_val_loss,
'Learning Rate': lr,
'Weight Decay': weight_decay,
'Drop out Rate': drop_out,
'Pearson': pears,
'Pearson Some': pears_some,
'Kappa': kappa,
'Kappa Some': kappa_some,
'Pearson (Anger)': pear_anger,
'Pearson some (Anger)': pear_some_anger,
'Kappa (Anger)': kappa_anger,
'Kappa some (Anger)': kappa_some_anger,
'Pearson (Fear)': pear_fear,
'Pearson some (Fear)': pear_some_fear,
'Kappa (Fear)': kappa_fear,
'Kappa some (Fear)': kappa_some_fear,
'Pearson (Joy)': pear_joy,
'Pearson some (Joy)': pear_some_joy,
'Kappa (Joy)': kappa_joy,
'Kappa some (Joy)': kappa_some_joy,
'Pearson (Sadness)': pear_sadness,
'Pearson some (Sadness)': pear_some_sadness,
'Kappa (Sadness)': kappa_sadness,
'Kappa some (Sadness)': kappa_some_sadness,
'Training Time': training_time,
'Validation Time': val_time,
})
return training_stats