def preprocessing(self, discourse):
for paragraph in discourse:
for sentence in paragraph.iterfind(
filter=node_type_filter(Sentence)):
if self.ctb and (sentence.sid is not None) and (sentence.sid
in self.ctb):
parse = self.ctb[sentence.sid]
pairs = [(node[0], node.label())
for node in parse.subtrees()
if node.height() == 2 and node.label() != "-NONE-"
]
words, tags = list(zip(*pairs))
else:
words, tags = list(zip(*thulac.cut(sentence.text)))
setattr(sentence, "words", list(words))
setattr(sentence, "tags", list(tags))
offset = 0
for textnode in sentence.iterfind(
filter=node_type_filter([TEXT, Connective, EDU]),
terminal=node_type_filter([TEXT, Connective, EDU])):
if isinstance(textnode, EDU):
edu_words = []
edu_tags = []
cur = 0
for word, tag in zip(sentence.words, sentence.tags):
if offset <= cur < cur + len(word) <= offset + len(
textnode.text):
edu_words.append(word)
edu_tags.append(tag)
cur += len(word)
setattr(textnode, "words", edu_words)
setattr(textnode, "tags", edu_tags)
offset += len(textnode.text)
return discourse
def gen_instances(trees):
instances = []
for tree in trees:
root = tree.root_relation()
if root is not None:
words = []
poses = []
for edu in root.iterfind(node_type_filter(EDU)):
words.append(edu.words)
poses.append(edu.tags)
trans = oracle(tree)
instances.append((words, poses, trans))
return instances
def evaluate(args):
pipeline = build_pipeline(schema=args.schema,
segmenter_name=args.segmenter_name,
use_gpu=args.use_gpu)
cdtb = CDTB(args.data,
"TRAIN",
"VALIDATE",
"TEST",
ctb_dir=args.ctb_dir,
preprocess=True,
cache_dir=args.cache_dir)
golds = list(filter(lambda d: d.root_relation(), chain(*cdtb.test)))
parses = []
if args.use_gold_edu:
logger.info("evaluation with gold edu segmentation")
else:
logger.info("evaluation with auto edu segmentation")
for para in tqdm(golds, desc="parsing", unit=" para"):
if args.use_gold_edu:
edus = []
for edu in para.edus():
edu_copy = EDU([TEXT(edu.text)])
setattr(edu_copy, "words", edu.words)
setattr(edu_copy, "tags", edu.tags)
edus.append(edu_copy)
else:
sentences = []
for sentence in para.sentences():
if list(sentence.iterfind(node_type_filter(EDU))):
copy_sentence = Sentence([TEXT([sentence.text])])
if hasattr(sentence, "words"):
setattr(copy_sentence, "words", sentence.words)
if hasattr(sentence, "tags"):
setattr(copy_sentence, "tags", sentence.tags)
setattr(copy_sentence, "parse", cdtb.ctb[sentence.sid])
sentences.append(copy_sentence)
para = pipeline.cut_edu(Paragraph(sentences))
edus = []
for edu in para.edus():
edu_copy = EDU([TEXT(edu.text)])
setattr(edu_copy, "words", edu.words)
setattr(edu_copy, "tags", edu.tags)
edus.append(edu_copy)
parse = pipeline.parse(Paragraph(edus))
parses.append(parse)
# edu score
scores = edu_eval(golds, parses)
logger.info("EDU segmentation scores:")
logger.info(gen_edu_report(scores))
# parser score
cdtb_macro_scores = eval.parse_eval(parses, golds, average="macro")
logger.info("CDTB macro (strict) scores:")
logger.info(eval.gen_parse_report(*cdtb_macro_scores))
# nuclear scores
nuclear_scores = eval.nuclear_eval(parses, golds)
logger.info("nuclear scores:")
logger.info(eval.gen_category_report(nuclear_scores))
# relation scores
ctype_scores, ftype_scores = eval.relation_eval(parses, golds)
logger.info("coarse relation scores:")
logger.info(eval.gen_category_report(ctype_scores))
logger.info("fine relation scores:")
logger.info(eval.gen_category_report(ftype_scores))
# height eval
height_scores = eval.height_eval(parses, golds)
logger.info("structure precision by node height:")
logger.info(eval.gen_height_report(height_scores))
logger = logging.getLogger("test rnn segmenter")
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
with open("data/models/segmenter.rnn.model", "rb") as model_fd:
model = torch.load(model_fd, map_location='cpu')
model.use_gpu = False
model.eval()
segmenter = RNNSegmenter(model)
cdtb = CDTB("data/CDTB",
"TRAIN",
"VALIDATE",
"TEST",
ctb_dir="data/CTB",
preprocess=True,
cache_dir="data/cache")
golds = []
segs = []
for paragraph in tqdm.tqdm(chain(*cdtb.test), desc="segmenting"):
seged_sents = []
for sentence in paragraph.sentences():
# make sure sentence has edus
if list(sentence.iterfind(node_type_filter(EDU))):
seged_sents.append(Sentence(segmenter.cut_edu(sentence)))
if seged_sents:
segs.append(Paragraph(seged_sents))
golds.append(paragraph)
scores = edu_eval(segs, golds)
logger.info(gen_edu_report(scores))
def evaluate(args):
with open("pub/models/segmenter.svm.model", "rb") as segmenter_fd:
segmenter_model = pickle.load(segmenter_fd)
with open("pub/models/treebuilder.partptr.model", "rb") as parser_fd:
parser_model = torch.load(parser_fd, map_location="cpu")
parser_model.use_gpu = False
parser_model.eval()
segmenter = SVMSegmenter(segmenter_model)
parser = PartPtrParser(parser_model)
cdtb = CDTB(args.data,
"TRAIN",
"VALIDATE",
"TEST",
ctb_dir=args.ctb_dir,
preprocess=True,
cache_dir=args.cache_dir)
golds = list(filter(lambda d: d.root_relation(), chain(*cdtb.test)))
parses = []
if args.use_gold_edu:
logger.info("evaluation with gold edu segmentation")
else:
logger.info("evaluation with auto edu segmentation")
for para in tqdm(golds, desc="parsing", unit=" para"):
if args.use_gold_edu:
edus = []
for edu in para.edus():
edu_copy = EDU([TEXT(edu.text)])
setattr(edu_copy, "words", edu.words)
setattr(edu_copy, "tags", edu.tags)
edus.append(edu_copy)
parse = parser.parse(Paragraph(edus))
parses.append(parse)
else:
edus = []
for sentence in para.sentences():
if list(sentence.iterfind(node_type_filter(EDU))):
setattr(sentence, "parse", cdtb.ctb[sentence.sid])
edus.extend(segmenter.cut_edu(sentence))
parse = parser.parse(Paragraph(edus))
parses.append(parse)
# edu score
scores = edu_eval(golds, parses)
logger.info("EDU segmentation scores:")
logger.info(gen_edu_report(scores))
# parser score
cdtb_macro_scores = eval.parse_eval(parses, golds, average="macro")
logger.info("CDTB macro (strict) scores:")
logger.info(eval.gen_parse_report(*cdtb_macro_scores))
# nuclear scores
nuclear_scores = eval.nuclear_eval(parses, golds)
logger.info("nuclear scores:")
logger.info(eval.gen_category_report(nuclear_scores))
# relation scores
ctype_scores, ftype_scores = eval.relation_eval(parses, golds)
logger.info("coarse relation scores:")
logger.info(eval.gen_category_report(ctype_scores))
logger.info("fine relation scores:")
logger.info(eval.gen_category_report(ftype_scores))
