def build_tree(self, edus, trans_probs):
buffer = deque(edus)
stack = []
for prob in trans_probs:
trans, nuclear, ftype = self.model.trans_label.id2label[prob.argmax()]
ctype = rev_relationmap[ftype] if ftype is not None else None
if trans == SHIFT:
stack.append(buffer.popleft())
elif trans == REDUCE:
right = stack.pop()
left = stack.pop()
comp = Relation([left, right], nuclear=nuclear, ftype=ftype, ctype=ctype)
stack.append(comp)
assert len(stack) == 1
return Paragraph([stack[0]])
def parse_and_eval(dataset, model):
parser = ShiftReduceParser(model)
golds = list(filter(lambda d: d.root_relation(), chain(*dataset)))
num_instances = len(golds)
strips = []
for paragraph in golds:
edus = []
for edu in paragraph.edus():
edu_copy = EDU([TEXT(edu.text)])
setattr(edu_copy, "words", edu.words)
setattr(edu_copy, "tags", edu.tags)
edus.append(edu_copy)
strips.append(Paragraph(edus))
parses = []
for strip in strips:
parses.append(parser.parse(strip))
return num_instances, parse_eval(parses, golds)
def evaluate(args):
pipeline = build_pipeline(schema=args.schema,
segmenter_name=args.segmenter_name,
use_gpu=args.use_gpu)
cdtb = CDTB(args.data,
"TRAIN",
"VALIDATE",
"TEST",
ctb_dir=args.ctb_dir,
preprocess=True,
cache_dir=args.cache_dir)
golds = list(filter(lambda d: d.root_relation(), chain(*cdtb.test)))
parses = []
if args.use_gold_edu:
logger.info("evaluation with gold edu segmentation")
else:
logger.info("evaluation with auto edu segmentation")
for para in tqdm(golds, desc="parsing", unit=" para"):
if args.use_gold_edu:
edus = []
for edu in para.edus():
edu_copy = EDU([TEXT(edu.text)])
setattr(edu_copy, "words", edu.words)
setattr(edu_copy, "tags", edu.tags)
edus.append(edu_copy)
else:
sentences = []
for sentence in para.sentences():
if list(sentence.iterfind(node_type_filter(EDU))):
copy_sentence = Sentence([TEXT([sentence.text])])
if hasattr(sentence, "words"):
setattr(copy_sentence, "words", sentence.words)
if hasattr(sentence, "tags"):
setattr(copy_sentence, "tags", sentence.tags)
setattr(copy_sentence, "parse", cdtb.ctb[sentence.sid])
sentences.append(copy_sentence)
para = pipeline.cut_edu(Paragraph(sentences))
edus = []
for edu in para.edus():
edu_copy = EDU([TEXT(edu.text)])
setattr(edu_copy, "words", edu.words)
setattr(edu_copy, "tags", edu.tags)
edus.append(edu_copy)
parse = pipeline.parse(Paragraph(edus))
parses.append(parse)
# edu score
scores = edu_eval(golds, parses)
logger.info("EDU segmentation scores:")
logger.info(gen_edu_report(scores))
# parser score
cdtb_macro_scores = eval.parse_eval(parses, golds, average="macro")
logger.info("CDTB macro (strict) scores:")
logger.info(eval.gen_parse_report(*cdtb_macro_scores))
# nuclear scores
nuclear_scores = eval.nuclear_eval(parses, golds)
logger.info("nuclear scores:")
logger.info(eval.gen_category_report(nuclear_scores))
# relation scores
ctype_scores, ftype_scores = eval.relation_eval(parses, golds)
logger.info("coarse relation scores:")
logger.info(eval.gen_category_report(ctype_scores))
logger.info("fine relation scores:")
logger.info(eval.gen_category_report(ftype_scores))
# height eval
height_scores = eval.height_eval(parses, golds)
logger.info("structure precision by node height:")
logger.info(eval.gen_height_report(height_scores))
logger = logging.getLogger("test rnn segmenter")
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
with open("data/models/segmenter.rnn.model", "rb") as model_fd:
model = torch.load(model_fd, map_location='cpu')
model.use_gpu = False
model.eval()
segmenter = RNNSegmenter(model)
cdtb = CDTB("data/CDTB",
"TRAIN",
"VALIDATE",
"TEST",
ctb_dir="data/CTB",
preprocess=True,
cache_dir="data/cache")
golds = []
segs = []
for paragraph in tqdm.tqdm(chain(*cdtb.test), desc="segmenting"):
seged_sents = []
for sentence in paragraph.sentences():
# make sure sentence has edus
if list(sentence.iterfind(node_type_filter(EDU))):
seged_sents.append(Sentence(segmenter.cut_edu(sentence)))
if seged_sents:
segs.append(Paragraph(seged_sents))
golds.append(paragraph)
scores = edu_eval(segs, golds)
logger.info(gen_edu_report(scores))
def cut_edu(self, para: Paragraph) -> Paragraph:
edus = []
for sentence in para.sentences():
edus.extend(self.segmenter.cut_edu(sentence))
return Paragraph(edus)
def cut_sent(self, text: str, sid=None):
return Paragraph(self.segmenter.cut_sent(text, sid=sid))
def cut(self, text):
sentences = self.cut_sent(text)
for i, sent in enumerate(sentences):
sentences[i] = Sentence(self.cut_edu(sent))
return Paragraph(sentences)
def evaluate(args):
with open("pub/models/segmenter.svm.model", "rb") as segmenter_fd:
segmenter_model = pickle.load(segmenter_fd)
with open("pub/models/treebuilder.partptr.model", "rb") as parser_fd:
parser_model = torch.load(parser_fd, map_location="cpu")
parser_model.use_gpu = False
parser_model.eval()
segmenter = SVMSegmenter(segmenter_model)
parser = PartPtrParser(parser_model)
cdtb = CDTB(args.data,
"TRAIN",
"VALIDATE",
"TEST",
ctb_dir=args.ctb_dir,
preprocess=True,
cache_dir=args.cache_dir)
golds = list(filter(lambda d: d.root_relation(), chain(*cdtb.test)))
parses = []
if args.use_gold_edu:
logger.info("evaluation with gold edu segmentation")
else:
logger.info("evaluation with auto edu segmentation")
for para in tqdm(golds, desc="parsing", unit=" para"):
if args.use_gold_edu:
edus = []
for edu in para.edus():
edu_copy = EDU([TEXT(edu.text)])
setattr(edu_copy, "words", edu.words)
setattr(edu_copy, "tags", edu.tags)
edus.append(edu_copy)
parse = parser.parse(Paragraph(edus))
parses.append(parse)
else:
edus = []
for sentence in para.sentences():
if list(sentence.iterfind(node_type_filter(EDU))):
setattr(sentence, "parse", cdtb.ctb[sentence.sid])
edus.extend(segmenter.cut_edu(sentence))
parse = parser.parse(Paragraph(edus))
parses.append(parse)
# edu score
scores = edu_eval(golds, parses)
logger.info("EDU segmentation scores:")
logger.info(gen_edu_report(scores))
# parser score
cdtb_macro_scores = eval.parse_eval(parses, golds, average="macro")
logger.info("CDTB macro (strict) scores:")
logger.info(eval.gen_parse_report(*cdtb_macro_scores))
# nuclear scores
nuclear_scores = eval.nuclear_eval(parses, golds)
logger.info("nuclear scores:")
logger.info(eval.gen_category_report(nuclear_scores))
# relation scores
ctype_scores, ftype_scores = eval.relation_eval(parses, golds)
logger.info("coarse relation scores:")
logger.info(eval.gen_category_report(ctype_scores))
logger.info("fine relation scores:")
logger.info(eval.gen_category_report(ftype_scores))
