def evaluate(model,
data,
loss_function,
feature_to_idx,
label_to_idx,
case_idx=0):
model.eval()
avg_loss = 0.0
truth_res = []
pred_res = []
for item in data:
events = item['events']
label = item['los_group']
truth_res.append(label_to_idx[label])
# detaching it from its history on the last instance.
model.hidden = model.init_hidden()
events = data_loader.prepare_sequence(events, feature_to_idx)
label = data_loader.prepare_label(label, label_to_idx)
pred = model(events)
pred_label = pred.data.max(1)[1].numpy()
pred_res.extend(pred_label)
loss = loss_function(pred, label)
avg_loss += loss.item()
avg_loss /= len(data)
scores = get_scores(truth_res, pred_res, prefix='test_', case_idx=case_idx)
logger.info(
'CASE=%s: Evaluate: \tavg_loss:%g \ttest acc: %s \ttest f1: %g ',
case_idx, avg_loss, scores['test_acc'], scores['test_f1'])
return scores
def train_epoch(model, train_data, loss_function, optimizer, word_to_ix, label_to_ix, i, device):
model.train()
avg_loss = 0.0
count = 0
truth_res = []
pred_res = []
batch_sent = []
for sent, label in train_data:
truth_res.append(label_to_ix[label])
# detaching it from its history on the last instance.
model.hidden = model.init_hidden(device)
sent = data_loader.prepare_sequence(sent, word_to_ix)
sent =sent.to(device)
label = data_loader.prepare_label(label, label_to_ix).to(device)
pred = model(sent)
pred_label = pred.data.max(1)[1].cpu().numpy()
pred_res.append(pred_label)
model.zero_grad()
loss = loss_function(pred, label)
avg_loss += loss.data.item()
count += 1
# if count % 500 == 0:
# print('epoch: %d iterations: %d loss :%g' % (i, count, loss.data.item()))
loss.backward()
optimizer.step()
avg_loss /= len(train_data)
print('epoch: %d done! \n train avg_loss:%g , acc:%g'%(i, avg_loss, get_accuracy(truth_res,pred_res)))
def train_epoch(experiment, model, train_data, loss_function, optimizer,
word_to_ix, label_to_ix, i):
model.train()
avg_loss = 0.0
count = 0
truth_res = []
pred_res = []
batch_sent = []
for batch, (sent, label) in experiment.batch_loop(iterable=train_data):
truth_res.append(label_to_ix[label])
# detaching it from its history on the last instance.
model.hidden = model.init_hidden()
sent = data_loader.prepare_sequence(sent, word_to_ix)
label = data_loader.prepare_label(label, label_to_ix)
pred = model(sent)
pred_label = pred.data.max(1)[1].numpy()
pred_res.append(pred_label)
model.zero_grad()
loss = loss_function(pred, label)
avg_loss += loss.data[0]
count += 1
if count % 500 == 0:
print('epoch: %d iterations: %d loss :%g' %
(i, count, loss.data[0]))
loss.backward()
optimizer.step()
avg_loss /= len(train_data)
global wrapped_accuracy_function
print('epoch: %d done! \n train avg_loss:%g , acc:%g' %
(i, avg_loss, wrapped_accuracy_function(truth_res, pred_res)))
def evaluate(model, data, loss_function, word_to_ix, label_to_ix, device, name ='dev'):
model.eval()
avg_loss = 0.0
truth_res = []
pred_res = []
for sent, label in data:
truth_res.append(label_to_ix[label])
# detaching it from its history on the last instance.
model.hidden = model.init_hidden(device)
sent = data_loader.prepare_sequence(sent, word_to_ix).to(device)
label = data_loader.prepare_label(label, label_to_ix).to(device)
pred = model(sent)
pred_label = pred.data.max(1)[1].cpu().numpy()
pred_res.append(pred_label)
# model.zero_grad() # should I keep this when I am evaluating the model?
loss = loss_function(pred, label)
avg_loss += loss.data.item()
avg_loss /= len(data)
acc = get_accuracy(truth_res, pred_res)
print(name + ' avg_loss:%g train acc:%g' % (avg_loss, acc ))
return acc
def evaluate(experiment,
model,
data,
loss_function,
word_to_ix,
label_to_ix,
name='dev'):
model.eval()
avg_loss = 0.0
truth_res = []
pred_res = []
if name == 'dev':
context = experiment.validation()
else:
context = experiment.test(test_iterations=len(data))
with context:
for sent, label in data:
truth_res.append(label_to_ix[label])
# detaching it from its history on the last instance.
model.hidden = model.init_hidden()
sent = data_loader.prepare_sequence(sent, word_to_ix)
label = data_loader.prepare_label(label, label_to_ix)
pred = model(sent)
pred_label = pred.data.max(1)[1].numpy()
pred_res.append(pred_label)
# model.zero_grad() # should I keep this when I am evaluating the model?
loss = loss_function(pred, label)
avg_loss += loss.data[0]
if name != 'dev':
experiment.confusion_matrix(target=label, output=pred)
avg_loss /= len(data)
global wrapped_accuracy_function
acc = wrapped_accuracy_function(truth_res, pred_res)
print(name + ' avg_loss:%g train acc:%g' % (avg_loss, acc))
return acc
def train_epoch(model,
train_data,
loss_function,
optimizer,
feature_to_idx,
label_to_idx,
epoch_th,
lr,
clip,
case_idx=0):
"""Summary
Args:
model (TYPE): Description
train_data (TYPE): Description
loss_function (TYPE): Description
optimizer (TYPE): Description
feature_to_idx (TYPE): Description
label_to_idx (TYPE): Description
epoch_th (TYPE): Description
Returns:
DICT: {'epoch': 9, 'avg_loss': 0.8, 'train_acc': 0.7}
"""
model.train()
avg_loss = 0.0
count = 0
truth_res = []
pred_res = []
batch_sent = []
for item in train_data:
events = item['events']
label = item['los_group']
truth_res.append(label_to_idx[label])
# detaching it from its history on the last instance.
model.hidden = model.init_hidden()
events = data_loader.prepare_sequence(events, feature_to_idx)
label = data_loader.prepare_label(label, label_to_idx)
pred = model(events)
pred_label = pred.data.max(1)[1].numpy()
# logger.debug('predict result: %s, pred_label: %s', pred.data, pred_label)
pred_res.extend(pred_label)
model.zero_grad()
loss = loss_function(pred, label)
avg_loss += loss.item()
count += 1
if count % 500 == 0:
logger.info('CASE=%s: Epoch: %d \titerations: %d \tloss: %g',
case_idx, epoch_th, count, loss.item())
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
if model.non_trainable is False:
nn.utils.clip_grad_norm_(model.parameters(), clip)
for p in model.parameters():
# re-check if this model could be trainable
if p.grad is None:
break
p.data.add_(-lr, p.grad.data)
optimizer.step()
avg_loss /= len(train_data)
# logger.debug('truth_res: %s', truth_res)
# logger.debug('pred_res: %s', pred_res)
scores = get_scores(truth_res,
pred_res,
prefix='train_',
case_idx=case_idx)
scores.update({'epoch': epoch_th, 'avg_loss': avg_loss})
logger.info(
'CASE=%s: Epoch: %d done. \tavg_loss: %g \ttrain_acc: %g \ttrain_f1: %g',
case_idx, epoch_th, avg_loss, scores['train_acc'], scores['train_f1'])
return scores, avg_loss