def preprocessing(self, discourse): for paragraph in discourse: for sentence in paragraph.iterfind( filter=node_type_filter(Sentence)): if self.ctb and (sentence.sid is not None) and (sentence.sid in self.ctb): parse = self.ctb[sentence.sid] pairs = [(node[0], node.label()) for node in parse.subtrees() if node.height() == 2 and node.label() != "-NONE-" ] words, tags = list(zip(*pairs)) else: words, tags = list(zip(*thulac.cut(sentence.text))) setattr(sentence, "words", list(words)) setattr(sentence, "tags", list(tags)) offset = 0 for textnode in sentence.iterfind( filter=node_type_filter([TEXT, Connective, EDU]), terminal=node_type_filter([TEXT, Connective, EDU])): if isinstance(textnode, EDU): edu_words = [] edu_tags = [] cur = 0 for word, tag in zip(sentence.words, sentence.tags): if offset <= cur < cur + len(word) <= offset + len( textnode.text): edu_words.append(word) edu_tags.append(tag) cur += len(word) setattr(textnode, "words", edu_words) setattr(textnode, "tags", edu_tags) offset += len(textnode.text) return discourse
def gen_instances(trees): instances = [] for tree in trees: root = tree.root_relation() if root is not None: words = [] poses = [] for edu in root.iterfind(node_type_filter(EDU)): words.append(edu.words) poses.append(edu.tags) trans = oracle(tree) instances.append((words, poses, trans)) return instances
def evaluate(args): pipeline = build_pipeline(schema=args.schema, segmenter_name=args.segmenter_name, use_gpu=args.use_gpu) cdtb = CDTB(args.data, "TRAIN", "VALIDATE", "TEST", ctb_dir=args.ctb_dir, preprocess=True, cache_dir=args.cache_dir) golds = list(filter(lambda d: d.root_relation(), chain(*cdtb.test))) parses = [] if args.use_gold_edu: logger.info("evaluation with gold edu segmentation") else: logger.info("evaluation with auto edu segmentation") for para in tqdm(golds, desc="parsing", unit=" para"): if args.use_gold_edu: edus = [] for edu in para.edus(): edu_copy = EDU([TEXT(edu.text)]) setattr(edu_copy, "words", edu.words) setattr(edu_copy, "tags", edu.tags) edus.append(edu_copy) else: sentences = [] for sentence in para.sentences(): if list(sentence.iterfind(node_type_filter(EDU))): copy_sentence = Sentence([TEXT([sentence.text])]) if hasattr(sentence, "words"): setattr(copy_sentence, "words", sentence.words) if hasattr(sentence, "tags"): setattr(copy_sentence, "tags", sentence.tags) setattr(copy_sentence, "parse", cdtb.ctb[sentence.sid]) sentences.append(copy_sentence) para = pipeline.cut_edu(Paragraph(sentences)) edus = [] for edu in para.edus(): edu_copy = EDU([TEXT(edu.text)]) setattr(edu_copy, "words", edu.words) setattr(edu_copy, "tags", edu.tags) edus.append(edu_copy) parse = pipeline.parse(Paragraph(edus)) parses.append(parse) # edu score scores = edu_eval(golds, parses) logger.info("EDU segmentation scores:") logger.info(gen_edu_report(scores)) # parser score cdtb_macro_scores = eval.parse_eval(parses, golds, average="macro") logger.info("CDTB macro (strict) scores:") logger.info(eval.gen_parse_report(*cdtb_macro_scores)) # nuclear scores nuclear_scores = eval.nuclear_eval(parses, golds) logger.info("nuclear scores:") logger.info(eval.gen_category_report(nuclear_scores)) # relation scores ctype_scores, ftype_scores = eval.relation_eval(parses, golds) logger.info("coarse relation scores:") logger.info(eval.gen_category_report(ctype_scores)) logger.info("fine relation scores:") logger.info(eval.gen_category_report(ftype_scores)) # height eval height_scores = eval.height_eval(parses, golds) logger.info("structure precision by node height:") logger.info(eval.gen_height_report(height_scores))
logger = logging.getLogger("test rnn segmenter") if __name__ == '__main__': logging.basicConfig(level=logging.INFO) with open("data/models/segmenter.rnn.model", "rb") as model_fd: model = torch.load(model_fd, map_location='cpu') model.use_gpu = False model.eval() segmenter = RNNSegmenter(model) cdtb = CDTB("data/CDTB", "TRAIN", "VALIDATE", "TEST", ctb_dir="data/CTB", preprocess=True, cache_dir="data/cache") golds = [] segs = [] for paragraph in tqdm.tqdm(chain(*cdtb.test), desc="segmenting"): seged_sents = [] for sentence in paragraph.sentences(): # make sure sentence has edus if list(sentence.iterfind(node_type_filter(EDU))): seged_sents.append(Sentence(segmenter.cut_edu(sentence))) if seged_sents: segs.append(Paragraph(seged_sents)) golds.append(paragraph) scores = edu_eval(segs, golds) logger.info(gen_edu_report(scores))
def evaluate(args): with open("pub/models/segmenter.svm.model", "rb") as segmenter_fd: segmenter_model = pickle.load(segmenter_fd) with open("pub/models/treebuilder.partptr.model", "rb") as parser_fd: parser_model = torch.load(parser_fd, map_location="cpu") parser_model.use_gpu = False parser_model.eval() segmenter = SVMSegmenter(segmenter_model) parser = PartPtrParser(parser_model) cdtb = CDTB(args.data, "TRAIN", "VALIDATE", "TEST", ctb_dir=args.ctb_dir, preprocess=True, cache_dir=args.cache_dir) golds = list(filter(lambda d: d.root_relation(), chain(*cdtb.test))) parses = [] if args.use_gold_edu: logger.info("evaluation with gold edu segmentation") else: logger.info("evaluation with auto edu segmentation") for para in tqdm(golds, desc="parsing", unit=" para"): if args.use_gold_edu: edus = [] for edu in para.edus(): edu_copy = EDU([TEXT(edu.text)]) setattr(edu_copy, "words", edu.words) setattr(edu_copy, "tags", edu.tags) edus.append(edu_copy) parse = parser.parse(Paragraph(edus)) parses.append(parse) else: edus = [] for sentence in para.sentences(): if list(sentence.iterfind(node_type_filter(EDU))): setattr(sentence, "parse", cdtb.ctb[sentence.sid]) edus.extend(segmenter.cut_edu(sentence)) parse = parser.parse(Paragraph(edus)) parses.append(parse) # edu score scores = edu_eval(golds, parses) logger.info("EDU segmentation scores:") logger.info(gen_edu_report(scores)) # parser score cdtb_macro_scores = eval.parse_eval(parses, golds, average="macro") logger.info("CDTB macro (strict) scores:") logger.info(eval.gen_parse_report(*cdtb_macro_scores)) # nuclear scores nuclear_scores = eval.nuclear_eval(parses, golds) logger.info("nuclear scores:") logger.info(eval.gen_category_report(nuclear_scores)) # relation scores ctype_scores, ftype_scores = eval.relation_eval(parses, golds) logger.info("coarse relation scores:") logger.info(eval.gen_category_report(ctype_scores)) logger.info("fine relation scores:") logger.info(eval.gen_category_report(ftype_scores))