def predict(model, loader): model.eval() preds = {'trees': [], 'probs': []} for words, *feats, trees, charts in loader: word_mask = words.ne(args.pad_index)[:, 1:] mask = word_mask if len(words.shape) < 3 else word_mask.any(-1) mask = (mask.unsqueeze(1) & mask.unsqueeze(2)).triu_(1) s_feat = model(words, feats) s_feat = model.crf(s_feat, mask, require_marginals=True) chart_preds = model.decode(s_feat, mask) preds['trees'].extend([ Tree.build(tree, [(i, j, CHART.vocab[label]) for i, j, label in chart]) for tree, chart in zip(trees, chart_preds) ]) if args.draw_pred: ### draw trees here filter_delete = lambda x: [it for it in x if it not in args.delete] trees_fact = [ Tree.factorize(tree, args.delete, args.equal) for tree in preds['trees'] ] leaves = [filter_delete(tree.leaves()) for tree in trees] t = convert_to_viz_tree(tree=trees_fact[0], sen=leaves[0]) draw_tree(t, res_path="./prediction") return preds
def evaluate(model, loader): model.eval() total_loss, metric = 0, SpanMetric() for words, *feats, trees, charts in loader: # mask out the lower left triangle word_mask = words.ne(args.pad_index)[:, 1:] mask = word_mask if len(words.shape) < 3 else word_mask.any(-1) mask = (mask.unsqueeze(1) & mask.unsqueeze(2)).triu_(1) s_feat = model(words, feats) loss, s_feat = model.loss(s_feat, charts, mask, require_marginals=True) chart_preds = model.decode(s_feat, mask) # since the evaluation relies on terminals, # the tree should be first built and then factorized preds = [ Tree.build(tree, [(i, j, CHART.vocab[label]) for i, j, label in chart]) for tree, chart in zip(trees, chart_preds) ] total_loss += loss.item() metric( [Tree.factorize(tree, args.delete, args.equal) for tree in preds], [Tree.factorize(tree, args.delete, args.equal) for tree in trees]) total_loss /= len(loader) return total_loss, metric
def evaluate(model, loader): model.eval() total_loss, metric = 0, SpanMetric() for words, *feats, trees, charts in loader: # mask out the lower left triangle word_mask = words.ne(args.pad_index)[:, 1:] mask = word_mask if len(words.shape) < 3 else word_mask.any(-1) mask = (mask.unsqueeze(1) & mask.unsqueeze(2)).triu_(1) s_feat = model(words, feats) loss, s_feat = model.loss(s_feat, charts, mask, require_marginals=True) chart_preds = model.decode(s_feat, mask) # since the evaluation relies on terminals, # the tree should be first built and then factorized preds = [ Tree.build(tree, [(i, j, CHART.vocab[label]) for i, j, label in chart]) for tree, chart in zip(trees, chart_preds) ] total_loss += loss.item() if args.draw_pred: ### draw trees here filter_delete = lambda x: [it for it in x if it not in args.delete] trees_fact = [ Tree.factorize(tree, args.delete, args.equal) for tree in preds ] leaves = [filter_delete(tree.leaves()) for tree in trees] t = convert_to_viz_tree(tree=trees_fact[0], sen=leaves[0]) draw_tree(t, res_path="./prediction") metric( [Tree.factorize(tree, args.delete, args.equal) for tree in preds], [Tree.factorize(tree, args.delete, args.equal) for tree in trees]) total_loss /= len(loader) return total_loss, metric
def predict(model, loader): model.eval() preds = {'trees': [], 'probs': []} for words, *feats, trees in loader: word_mask = words.ne(args.pad_index)[:, 1:] mask = word_mask if len(words.shape) < 3 else word_mask.any(-1) mask = (mask.unsqueeze(1) & mask.unsqueeze(2)).triu_(1) lens = mask[:, 0].sum(-1) s_feat = model(words, feats) s_span = model.crf(s_feat, mask, require_marginals=True) chart_preds = model.decode(s_span, mask) preds['trees'].extend([ Tree.build(tree, [(i, j, CHART.vocab[label]) for i, j, label in chart]) for tree, chart in zip(trees, chart_preds) ]) if args.prob: preds['probs'].extend( [prob[:i - 1, 1:i].cpu() for i, prob in zip(lens, s_span)]) return preds