def evaluate(config: Transformer,
model: Transformer,
eval_loader: DataLoader,
device='cpu'):
"""evaluate function"""
model.eval()
with torch.no_grad():
total_eval_loss, n_word_correct, n_word_total = 0.0, 0, 0
for ids, sample in enumerate(tqdm(eval_loader)):
input_ids, decoder_input_ids, decoder_target_ids = (
sample['input_ids'].to(device),
sample['decode_input_ids'].to(device),
sample['decode_label_ids'].to(device))
logits = model(input_ids, decoder_input_ids)
loss, n_correct, n_word = cal_performance(
logits,
gold=decoder_target_ids,
trg_pad_idx=config.pad_idx,
smoothing=config.label_smoothing)
total_eval_loss += loss.item()
n_word_correct += n_correct
n_word_total += n_word
average_loss = total_eval_loss / n_word_total
accuracy = n_word_correct / n_word_total
return average_loss, accuracy
def __init__(self, src_vocab, tgt_vocab, checkpoint, opts):
self.src_vocab = src_vocab
self.tgt_vocab = tgt_vocab
hparams = checkpoint['hparams']
transformer = Transformer(len(src_vocab),
len(tgt_vocab),
hparams.max_len + 2,
n_layers=hparams.n_layers,
d_model=hparams.d_model,
d_emb=hparams.d_model,
d_hidden=hparams.d_hidden,
n_heads=hparams.n_heads,
d_k=hparams.d_k,
d_v=hparams.d_v,
dropout=hparams.dropout,
pad_id=src_vocab.pad_id)
transformer.load_state_dict(checkpoint['model'])
log_proj = torch.nn.LogSoftmax()
if hparams.cuda:
transformer.cuda()
log_proj.cuda()
transformer.eval()
self.hparams = hparams
self.opts = opts
self.model = transformer
self.log_proj = log_proj
def evaluate_epoch(opt: Namespace, model: Transformer, val_data):
model.eval()
start_time = datetime.now()
total_loss = total_word = total_corrected_word = 0
with torch.no_grad():
for i, batch in tqdm(enumerate(val_data),
total=len(val_data),
leave=False):
# Prepare validation data
src_input, trg_input, y_true = _prepare_batch_data(
batch, opt.device)
# Forward
y_pred = model(src_input, trg_input)
loss = calculate_loss(y_pred, y_true, opt.trg_pad_idx)
n_word, n_corrected = calculate_performance(
y_pred, y_true, opt.trg_pad_idx)
# Validation Logs
total_loss += loss.item()
total_word += n_word
total_corrected_word += n_corrected
loss_per_word = total_loss / total_word
accuracy = total_corrected_word / total_word
return {
'total_seconds': (datetime.now() - start_time).total_seconds(),
'total_loss': total_loss,
'total_word': total_word,
'total_corrected_word': total_corrected_word,
'loss_per_word': loss_per_word,
'accuracy': accuracy
}