def evaluate(model,
test_data,
model_config,
other_config,
loss_criterion,
final_eval=False):
P, T, probs = [], [], []
with torch.no_grad():
acc, batch_count, loss = 0, 0, 0
actual_sentences, predicted_sentences = [], []
for batch in test_data:
sentence, pos, tags = batch[0][0].to(dp.device), batch[0][1].to(
dp.device), batch[1].to(dp.device)
sentence_len = len(tags)
hidden = model.hidden
if 'crf' in model_config and model_config['crf'] == True:
l, feats = model(sentence, pos, hidden, tags)
path_score, decoded_best_path = model.decode(feats)
predicted = [i.item() for i in decoded_best_path]
else:
model_output = model(sentence, pos, hidden, tags)
topv, topi = model_output.topk(1)
predicted = [i.item() for i in topi]
l = loss_criterion(model_output, tags.view(tags.size(0)))
proba_pred = topv.to('cpu').numpy()
for i in proba_pred[:, 0]:
probs.append(i)
target = [i.item() for i in tags]
target_sentence, predicted_sentence = utils.extract_predictions_from_model_output(
target, predicted, other_config['index2tag'])
actual_sentences.append(target_sentence)
predicted_sentences.append(predicted_sentence)
# accuracy
acc += utils.token_level_accuracy(target, predicted)
batch_count += 1
# loss
loss += np.round(l.item(), 4)
P.append(predicted_sentence.split())
T.append(target_sentence.split())
evaluation_accuracy = float(acc / batch_count)
evaluation_loss = float(loss / batch_count)
P = [i for j in P for i in j]
T = [i for j in T for i in j]
classes = [v for k, v in other_config['index2tag'].items()]
F_measure, AP, fig, classification = utils.metrics_score(target=T,
predicted=P,
cls=classes,
prfs=True,
probs=probs)
if final_eval:
return actual_sentences, predicted_sentences, evaluation_accuracy, F_measure, AP, classification, fig
return evaluation_accuracy, evaluation_loss, F_measure, AP
def evaluate(model, test_data, data_dir, index2tag):
P, T, probs = [], [], []
with torch.no_grad():
with open(os.path.join(data_dir, 'decoded.out'), 'w') as d, open(
os.path.join(data_dir, 'baseline_model_results.txt'),
'w') as h:
acc, acc_count = 0, 0
for sent, tags in test_data:
score, predicted = model(sent)
target = [i.item() for i in tags]
target_sentence, predicted_sentence = utils.extract_predictions_from_model_output(
target, predicted, index2tag)
for u, v in zip(target_sentence.split(),
predicted_sentence.split()):
d.write('{} {}'.format(u.strip(), v.strip()))
d.write('\n')
d.write('\n')
acc += utils.token_level_accuracy(target, predicted)
acc_count += 1
P.append(predicted_sentence.split())
T.append(target_sentence.split())
evaluation_accuracy = float(acc / acc_count)
print('Evaluation accuracy {}'.format(evaluation_accuracy))
P = [i for j in P for i in j]
T = [i for j in T for i in j]
classes = [v for k, v in index2tag.items()]
F_measure, AP, fig, classification = utils.metrics_score(
target=T, predicted=P, cls=classes, prfs=True, probs=probs)
h.write('Classification Matrix \n {} \n'.format(classification))
h.write('Average Precision: {} \n'.format(AP))
d.close()
h.close()
print(classification)
return evaluation_accuracy, classification, fig
def train(encoder,
decoder,
epochs,
input_words,
output_tags,
max_len,
learning_rate,
print_every,
batch_size,
use_teacher_forcing=False):
losses, accuracies = [], []
print_loss_total = 0
plot_loss_total = 0
#define backprop strategy
encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)
decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)
loss_criterion = nn.NLLLoss()
start = time.time()
epoch_loss, epoch_accuracy = 0, 0
for epoch in range(epochs):
training_tensors = list(zip(input_words, output_tags))
epoch_batch_loss, epoch_batch_acc = 0, 0
for batch_pairs in utils.batch_generator(training_tensors, batch_size):
batch_loss, batch_acc = 0, 0
for batch_tensor in batch_pairs:
input_tensor = batch_tensor[0].to(device)
target_tensor = batch_tensor[1].to(device)
encoder_optimizer.zero_grad()
decoder_optimizer.zero_grad()
input_len, target_len = input_tensor.size(
0), target_tensor.size(0)
encoder_hidden = encoder.initHidden()
encoder_outputs = torch.zeros(max_len,
encoder.hidden_size,
device=device)
loss = 0
for e in range(input_len):
encoder_output, out_encoder_hidden = encoder(
input_tensor[e], encoder_hidden)
encoder_outputs[e] = encoder_output[0, 0]
decoder_hidden = out_encoder_hidden
decoder_input = torch.tensor([[start_outcome_token]],
device=device)
predicted_tags = []
if use_teacher_forcing:
for d in range(target_len):
decoder_output, decoder_hidden = decoder(
decoder_input, decoder_hidden, encoder_outputs)
loss += loss_criterion(decoder_output,
target_tensor[d])
decoder_input = target_tensor[d]
else:
for d in range(target_len):
decoder_output, out_decoder_hidden, attn_weights = decoder(
decoder_input, decoder_hidden, encoder_outputs)
topv, topi = decoder_output.topk(1)
decoder_input = topi.squeeze().detach()
loss += loss_criterion(decoder_output,
target_tensor[d])
if decoder_input.item() == end_outcome_token:
break
else:
predicted_tags.append(topi.item())
loss.backward()
encoder_optimizer.step()
decoder_optimizer.step()
#loss
batch_loss += (loss.item() / target_len)
#accuracy
true_tags = [i.item() for i in target_tensor][:-1]
predicted_tags = predicted_tags[1:]
sim, tot = utils.token_level_accuracy(true_tags,
predicted_tags)
batch_acc += sim / tot
epoch_batch_loss += (batch_loss / len(batch_pairs))
epoch_batch_acc += (batch_acc / len(batch_pairs))
print(
"AVERAGE BATCH_LOSS {} AVERAGE TOKEN LEVEL ACCURACY {}".format(
(batch_loss / len(batch_pairs)),
(batch_acc / len(batch_pairs))))
epoch_loss += (epoch_batch_loss / batch_size)
epoch_accuracy += (epoch_batch_acc / batch_size)
losses.append(epoch_batch_loss / batch_size)
accuracies.append(epoch_batch_acc / batch_size)
print('EPOCH_LOSS {}'.format(epoch_batch_loss / batch_size))
if (epoch + 1) % print_every == 0:
now = time.time()
print(
'Epoch {}, Average Loss:- {}, Average Accuracy:- {}, Duration - {:.4f}'
.format((epoch + 1), (epoch_loss / print_every),
(epoch_accuracy / print_every), (now - start)))
epoch_loss = 0
return encoder, decoder, losses, accuracies
def obj_function(data_source,
sampling,
sampling_technique='under',
balanced_loss=False,
loss_method=None,
gamma=None,
tune=None):
#phase 1
if tune == 'phase1':
lrs = config['learning-rate']
bs = config['batch-size']
epoch_set = config['epochs']
parameters = list(it.product(lrs, bs, epoch_set))
elif tune == 'phase2':
parameters = config['sampling_percentage']
elif tune == 'phase3':
hidden_dim = config['hidden_dim']
embeddin_dim = config['embedding_dim']
dropout_rate = config['dropout_rate']
parameters = list(it.product(hidden_dim, embeddin_dim, dropout_rate))
start_outcome_token = 0
loss_criterion = nn.CrossEntropyLoss()
with open('parameter-performance-phase2.txt', 'w') as store_parameter:
for p in parameters:
if tune == 'phase1':
lr, b_s, epochs = p
us = [50]
elif tune == 'phase2':
lr, b_s, epochs = 0.1, 300, 60
us = [p]
elif tune == 'phase3':
lr, b_s, epochs = 0.1, 300, 60
us = [p]
# load data
model_configurations, batching_configurations, other_config = load_data(
data_source)
# load_model and beging training it on the loaded data
print('\nTRAINING BEGINS\n')
model = NNModel(model_configurations).to(dp.device)
# initialize weights
for param, values in model.named_parameters():
if param.__contains__('weight'):
torch.nn.init.xavier_normal_(values)
elif param.__contains__('bias'):
torch.nn.init.constant_(values, 0.0)
# define backprop strategy
optimizer = optim.SGD(model.parameters(), lr=lr)
epoch_loss, epoch_accuracy, epoch_count = 0, 0, 1
average_accuracy, average_loss = [], []
print('lr: {} batch_size {} epochs {} us {}'.format(
lr, b_s, epochs, us[0]))
store_parameter.write(
'lr: {} batch_size: {} epochs: {} us: {}'.format(
lr, b_s, epochs, us[0]))
for u in us:
# undersample raining data to create some balance
data = open('../ebm-data/dataset.pickle', 'rb')
data_loaded = pickle.load(data)
train_data, val_data, label_weights = data_loaded[
'train_full'], data_loaded['val_data'], data_loaded[
'weights_full']
#train_data, val_data, label_weights = batch_up_sets(batching_configurations, other_config, sampling=sampling, sampling_technique=sampling_technique)
target_weights = torch.FloatTensor(
[np.round(i, 5) for i in label_weights])
target_weights = target_weights.to(dp.device)
loss_calculator = nn.CrossEntropyLoss(
weight=target_weights
) if balanced_loss else nn.CrossEntropyLoss()
time_taken = 0
model.train()
for epoch in range(epochs):
batch_loss, batch_acc = 0, 0
batch_count = 0
start = time.time()
for batch_pairs in utils.batch_generator(
train_data[:4000], b_s):
acc, loss = 0, 0
for batch in batch_pairs:
model.zero_grad()
hidden = model.hidden
sentence, pos, tags = batch[0][0].to(
dp.device), batch[0][1].to(
dp.device), batch[1].to(dp.device)
if 'crf' in model_configurations and model_configurations[
'crf'] == True:
l, feats = model(sentence, pos, hidden, tags)
path_score, decoded_best_path = model.decode(
feats)
predicted = [
i.item() for i in decoded_best_path
]
else:
tag_scores = model(sentence, pos, hidden, tags)
if loss_method == 'focal' and gamma != None:
# print('+++++++++++++++++++Using focal loss+++++++++++++++++++++++++++++++++=')
l = utils.focal_loss(
tag_scores, tags.view(tags.size(0)),
target_weights, gamma)
else:
l = loss_calculator(
tag_scores, tags.view(tags.size(0)))
topv, topi = tag_scores.topk(1)
predicted = [i.item() for i in topi]
target = [i.item() for i in tags]
#accuracy
a = utils.token_level_accuracy(target, predicted)
acc += a
#loss
loss += np.round(l.item(), 4)
# print('accuracy {} Loss {}'.format(a, np.round(l.item(), 4)))
l.backward()
optimizer.step()
# print('Batch Average accuracy:- {:.3f} and Average loss:- {:.3f}'.format((acc / batch_size), (loss / batch_size)))
batch_acc += acc / b_s
batch_loss += loss / b_s
batch_count += 1
epoch_loss += batch_loss / batch_count
epoch_accuracy += batch_acc / batch_count
average_accuracy.append((batch_acc / batch_count))
average_loss.append((batch_loss / batch_count))
time_taken += (time.time() - start)
#print('Epoch {} acc {:.3f} and epoch loss {:.3f}. Duration {:.1f}'.format(epoch_count, (batch_acc / batch_count), (batch_loss / batch_count), (time.time() - start)))
epoch_count += 1
model.eval()
evaluation_accuracy, evaluation_loss, F_measure, AP = evaluate(
model,
val_data[:1000],
model_configurations,
other_config,
loss_calculator,
final_eval=False)
print(
'\nEvaluation accuracy {}, Evaluation loss {}, F1 score {}, Average Precision {}\n'
.format(evaluation_accuracy, evaluation_loss, F_measure,
AP))
store_parameter.write(
'Evaluation accuracy {}, Evaluation loss {}, F1 score {}, Average Precision {}\n'
.format(evaluation_accuracy, evaluation_loss, F_measure,
AP))
epoch_count += 1
return float(np.mean(average_accuracy)), float(np.mean(average_loss))
def train(model,
train_data,
model_config,
other_config,
pltdir,
args,
label_weights,
val_data,
balanced_loss=False,
loss_method=None,
gamma=None):
target_weights = torch.FloatTensor([np.round(i, 5) for i in label_weights])
target_weights = target_weights.to(dp.device)
calculate_loss = nn.CrossEntropyLoss(
weight=target_weights) if balanced_loss else nn.CrossEntropyLoss()
#required parameters
optimizer = optim.SGD(model.parameters(), lr=args.learning_rate)
#early_stopping = EarlyStopping(patience=patience, verbose=True)
average_accuracy, average_loss = [], []
epoch_loss, epoch_accuracy, epoch_count = 0, 0, 1
best_accuracy, best_model, F1_scores = 0, [], []
time_taken = 0
for epoch in range(args.n_epochs):
model.train()
batch_acc, batch_loss, batch_count = 0, 0, 0
start = time.time()
for batches in utils.batch_generator(train_data, args.batch_size):
acc, loss = 0, 0
# sente = [x for x,y in batches]
# sentences = [i[0] for i in sente]
# sent_pos = [i[1] for i in sente]
# tags = [y for x,y in batches]
# model.zero_grad()
# hidden = model.hidden
# tag_scores = model(sentences, sent_pos, hidden)
#
for batch in batches:
model.zero_grad()
hidden = model.hidden
sentence, pos, tags = batch[0][0].to(
dp.device), batch[0][1].to(dp.device), batch[1].to(
dp.device)
sd = [i.item() for i in sentence]
#print(' '.join([other_config['index2word'][i] for i in sd]))
if 'crf' in model_config and model_config['crf'] == True:
l, feats = model(sentence, pos, hidden, tags)
path_score, decoded_best_path = model.decode(feats)
predicted = [i.item() for i in decoded_best_path]
else:
tag_scores = model(sentence, pos, hidden, tags)
if loss_method == 'focal' and gamma != None:
#print('+++++++++++++++++++Using focal loss+++++++++++++++++++++++++++++++++=')
l = utils.focal_loss(tag_scores,
tags.view(tags.size(0)),
target_weights, gamma)
else:
l = calculate_loss(tag_scores, tags.view(tags.size(0)))
topv, topi = tag_scores.topk(1)
predicted = [i.item() for i in topi]
target = [i.item() for i in tags]
a = utils.token_level_accuracy(target, predicted)
acc += a
loss += np.round(l.item(), 4)
l.backward()
optimizer.step()
print('Batch Average accuracy:- {:.3f} and Average loss:- {:.3f}'.
format((acc / args.batch_size), (loss / args.batch_size)))
batch_acc += acc / args.batch_size
batch_loss += loss / args.batch_size
batch_count += 1
epoch_loss += batch_loss / batch_count
epoch_accuracy += batch_acc / batch_count
average_accuracy.append((batch_acc / batch_count))
average_loss.append((batch_loss / batch_count))
time_taken += (time.time() - start)
print('Epoch {} acc {:.3f} and epoch loss {:.3f}. Duration {:.1f}'.
format(epoch_count, (batch_acc / batch_count) * 100,
(batch_loss / batch_count), (time.time() - start)))
if args.evaluate_during_training:
if epoch == (args.n_epochs - 1):
print('EVALUATION AFTER FINAL EPOCH')
actual_sentences, predicted_sentences, evaluation_accuracy, F_measure, AP, fig, classification = evaluate(
model,
val_data,
model_config,
other_config,
loss_criterion=calculate_loss,
final_eval=True)
with open(os.path.join(pltdir, 'eval_results.txt'), 'w') as h:
h.write('Validation Classification Matrix \n{}\n\n'.format(
classification))
h.write('Average Precision: {:.3f}\n'.format(AP))
h.write('F1 score: {:.3f}'.format(F_measure))
h.close()
torch.save(
model.state_dict(),
os.path.join(
pltdir,
os.path.basename(args.args.outputdir) + '.pth'))
print(
'Evaluation accuracy: {:.3f}, F1 score: {:.3f}, Average precision: {:.3f}'
.format(evaluation_accuracy * 100, F_measure * 100,
AP * 100))
print('Time taken: {}s'.format(time.time() - start))
else:
evaluation_accuracy, F_measure, AP, classification, fig = evaluate(
model,
val_data,
model_config,
other_config,
loss_criterion=calculate_loss,
final_eval=False)
print(
'Evaluation accuracy: {:.3f}, F1 score: {:.3f}, Average precision: {:.3f}'
.format(evaluation_accuracy * 100, F_measure * 100,
AP * 100))
print('Time taken: {:.1f}s'.format(time.time() - start))
F1_scores.append(F_measure)
epoch_count += 1
#visulaize training
utils.visualize_model(average_accuracy, average_loss,
os.path.basename(args.outputdir), pltdir)
print('Total time taken to train: {}s'.format(np.round(time_taken, 2)))
return model, calculate_loss