def _evaluate(self, loader):
self.model.eval()
total_loss, metric = 0, SpanMetric()
for batch in loader:
words, *feats, trees, charts = batch
word_mask = words.ne(self.args.pad_index)[:, 1:]
mask = word_mask if len(words.shape) < 3 else word_mask.any(-1)
mask = (mask.unsqueeze(1) & mask.unsqueeze(2)).triu_(1)
s_span, s_pair, s_label = self.model(words, feats)
loss, s_span = self.model.loss(s_span, s_pair, s_label, charts,
mask)
chart_preds = self.model.decode(s_span, s_label, mask)
# since the evaluation relies on terminals,
# the tree should be first built and then factorized
preds = [
Tree.build(tree, [(i, j, self.CHART.vocab[label])
for i, j, label in chart])
for tree, chart in zip(trees, chart_preds)
]
total_loss += loss.item()
metric([
Tree.factorize(tree, self.args.delete, self.args.equal)
for tree in preds
], [
Tree.factorize(tree, self.args.delete, self.args.equal)
for tree in trees
])
total_loss /= len(loader)
return total_loss, metric
def _evaluate(self, loader):
self.model.eval()
total_loss, metric = 0, BracketMetric()
for words, feats, trees, (spans, labels) in loader:
batch_size, seq_len = words.shape
lens = words.ne(self.args.pad_index).sum(1) - 1
mask = lens.new_tensor(range(seq_len - 1)) < lens.view(-1, 1, 1)
mask = mask & mask.new_ones(seq_len - 1, seq_len - 1).triu_(1)
s_span, s_label = self.model(words, feats)
loss, s_span = self.model.loss(s_span, s_label, spans, labels,
mask, self.args.mbr)
chart_preds = self.model.decode(s_span, s_label, mask)
# since the evaluation relies on terminals,
# the tree should be first built and then factorized
preds = [
Tree.build(tree, [(i, j, self.CHART.vocab[label])
for i, j, label in chart])
for tree, chart in zip(trees, chart_preds)
]
total_loss += loss.item()
metric([
Tree.factorize(tree, self.args.delete, self.args.equal)
for tree in preds
], [
Tree.factorize(tree, self.args.delete, self.args.equal)
for tree in trees
])
total_loss /= len(loader)
return total_loss, metric