def eval(self):
self.model.eval()
pred_result = {}
for _, batch in enumerate(self.dataloader_dev):
question_ids, question, question_char, question_mask, \
passage, passage_char, passage_mask, passage_tokenized = batch
begin_, end_ = self.model(question, question_char, question_mask, passage, passage_char,
passage_mask) # batch x seq
probability = torch.bmm(F.softmax(begin_, dim=-1).unsqueeze(2),
F.softmax(end_, dim=-1).unsqueeze(1)).cpu().numpy()
mask = np.float32(np.fromfunction(lambda k, i, j: (i - j <= 15) & (i <= j),
(1, probability.shape[1], probability.shape[2])))
probability = mask * probability
pred_begin = np.argmax(np.amax(probability, axis=2), axis=1)
pred_end = np.argmax(np.amax(probability, axis=1), axis=1)
for i, (begin, end) in enumerate(zip(pred_begin, pred_end)):
ans = passage_tokenized[i][begin:end + 1]
qid = question_ids[i]
pred_result[qid] = " ".join(ans)
self.model.train()
em, f1 = evaluate(self.dev_dataset, pred_result)
self.scheduler.step(em)
return em, f1
def eval(self, args):
self.model.eval()
pred_result = {}
for _, batch in enumerate(self.dataloader_dev):
question_ids, questions, passages, passage_tokenized = batch
questions.variable(args, volatile=True)
passages.variable(args, volatile=True)
begin_, end_ = self.model(questions, passages) # batch x seq
_, pred_begin = torch.max(begin_, 1)
_, pred_end = torch.max(end_, 1)
pred = torch.stack([pred_begin, pred_end], dim=1)
for i, (begin, end) in enumerate(pred.cpu().data.numpy()):
ans = passage_tokenized[i][begin:end + 1]
qid = question_ids[i]
pred_result[qid] = " ".join(ans)
self.model.train()
return evaluate(self.dev_dataset, pred_result)