def evaluate(model, loader):
model.eval()
total_loss, metric = 0, SpanMetric()
for words, *feats, trees, charts in loader:
# mask out the lower left triangle
word_mask = words.ne(args.pad_index)[:, 1:]
mask = word_mask if len(words.shape) < 3 else word_mask.any(-1)
mask = (mask.unsqueeze(1) & mask.unsqueeze(2)).triu_(1)
s_feat = model(words, feats)
loss, s_feat = model.loss(s_feat, charts, mask, require_marginals=True)
chart_preds = model.decode(s_feat, mask)
# since the evaluation relies on terminals,
# the tree should be first built and then factorized
preds = [
Tree.build(tree, [(i, j, CHART.vocab[label])
for i, j, label in chart])
for tree, chart in zip(trees, chart_preds)
]
total_loss += loss.item()
metric(
[Tree.factorize(tree, args.delete, args.equal) for tree in preds],
[Tree.factorize(tree, args.delete, args.equal) for tree in trees])
total_loss /= len(loader)
return total_loss, metric
def predict(model, loader):
model.eval()
preds = {'trees': [], 'probs': []}
for words, *feats, trees, charts in loader:
word_mask = words.ne(args.pad_index)[:, 1:]
mask = word_mask if len(words.shape) < 3 else word_mask.any(-1)
mask = (mask.unsqueeze(1) & mask.unsqueeze(2)).triu_(1)
s_feat = model(words, feats)
s_feat = model.crf(s_feat, mask, require_marginals=True)
chart_preds = model.decode(s_feat, mask)
preds['trees'].extend([
Tree.build(tree, [(i, j, CHART.vocab[label])
for i, j, label in chart])
for tree, chart in zip(trees, chart_preds)
])
if args.draw_pred: ### draw trees here
filter_delete = lambda x: [it for it in x if it not in args.delete]
trees_fact = [
Tree.factorize(tree, args.delete, args.equal)
for tree in preds['trees']
]
leaves = [filter_delete(tree.leaves()) for tree in trees]
t = convert_to_viz_tree(tree=trees_fact[0], sen=leaves[0])
draw_tree(t, res_path="./prediction")
return preds
def evaluate(model, loader):
model.eval()
total_loss, metric = 0, SpanMetric()
for words, *feats, trees, charts in loader:
# mask out the lower left triangle
word_mask = words.ne(args.pad_index)[:, 1:]
mask = word_mask if len(words.shape) < 3 else word_mask.any(-1)
mask = (mask.unsqueeze(1) & mask.unsqueeze(2)).triu_(1)
s_feat = model(words, feats)
loss, s_feat = model.loss(s_feat, charts, mask, require_marginals=True)
chart_preds = model.decode(s_feat, mask)
# since the evaluation relies on terminals,
# the tree should be first built and then factorized
preds = [
Tree.build(tree, [(i, j, CHART.vocab[label])
for i, j, label in chart])
for tree, chart in zip(trees, chart_preds)
]
total_loss += loss.item()
if args.draw_pred: ### draw trees here
filter_delete = lambda x: [it for it in x if it not in args.delete]
trees_fact = [
Tree.factorize(tree, args.delete, args.equal) for tree in preds
]
leaves = [filter_delete(tree.leaves()) for tree in trees]
t = convert_to_viz_tree(tree=trees_fact[0], sen=leaves[0])
draw_tree(t, res_path="./prediction")
metric(
[Tree.factorize(tree, args.delete, args.equal) for tree in preds],
[Tree.factorize(tree, args.delete, args.equal) for tree in trees])
total_loss /= len(loader)
return total_loss, metric
