def read_rst_trees(data_dir): # Read RST tree file files = [os.path.join(data_dir, fname) for fname in os.listdir(data_dir) if fname.endswith('.dis')] for i, fdis in enumerate(files): fmerge = fdis.replace('.dis', '.merge') if not os.path.isfile(fmerge): print("Corresponding .fmerge file does not exist. Skipping the file.") continue rst_tree = RstTree(fdis, fmerge) rst_tree.build() yield rst_tree
def __getitem__(self, idx): """ Returns: If train: features for that tree (corefs, organizational features, spans, etc.) If test: gold tree """ if self.is_train: return self.X[idx], torch.LongTensor(self.y[idx]) doc = self.data_helper.docs[self.X[idx][0]] gold_rst = RstTree(doc.filename, doc.filename.replace('.dis', '.merge')) gold_rst.build() return gold_rst
def eval_parser(self, dev_data=None, path='./examples', use_parseval=False): """ Test the parsing performance""" # Evaluation met = Metrics(use_parseval, levels=['span', 'nuclearity']) # ---------------------------------------- # Read all files from the given path if dev_data is None: eval_list = [ os.path.join(path, fname) for fname in os.listdir(path) if fname.endswith('.merge') ] else: eval_list = dev_data pred_forms = [] gold_forms = [] total_cost = 0 for eval_instance in eval_list: # ---------------------------------------- # Read *.merge file doc = Doc() if dev_data is not None: gold_rst = eval_instance doc = eval_instance.doc else: doc.read_from_fmerge(eval_instance) eval_instance = eval_instance.replace('.dis', '.merge') fdis = eval_instance.replace('.merge', '.dis') gold_rst = RstTree(fdis, eval_instance) gold_rst.build() _, gold_action_seq = gold_rst.generate_action_samples() gold_action_seq = list( map(lambda x: self.data_helper.action_map[x], gold_action_seq)) pred_rst, cost = self.parser.sr_parse( [doc], [torch.cuda.LongTensor(gold_action_seq)], None, is_train=False) total_cost += cost pred_rst = pred_rst[0] # ---------------------------------------- # Evaluate with gold RST tree met.eval(gold_rst, pred_rst) print("Total cost: ", total_cost) met.report()
def read_rst_trees(data_dir, output_dir, parse_type, isFlat): # Read RST tree file files = [ os.path.join(data_dir, fname) for fname in os.listdir(data_dir) if fname.endswith('.dis') ] with open(os.path.join(output_dir, parse_type, "processed_data.p"), 'rb') as file: fmerges = pickle.load(file) rst_trees = [] for fdis, fmerge in zip(files, fmerges): print("building tree " + fdis) rst_tree = RstTree(fdis, fmerge, isFlat) rst_tree.build() rst_trees.append(rst_tree) return rst_trees
def test_eval(): fdis = "../data/data_dir/train_dir/wsj_0603.out.dis" fmerge = "../data/data_dir/train_dir/wsj_0603.out.merge" gold_tree = RstTree(fdis, fmerge) gold_tree.build() eval_trees(gold_tree, gold_tree, 1, 1, 1) sr_parser = ParsingState([], []) sr_parser.init(gold_tree.doc) sh, rd = 'Shift', 'Reduce' nn, ns, sn = 'NN', 'NS', 'SN' # Last two nuclearity actions are incorrect, last rel is incorrect silver_nuc = [(sh, None, None), (sh, None, None), (rd, ns, 'Elaboration'), (sh, None, None), (sh, None, None), (rd, ns, 'Elaboration'), (rd, nn, 'Same-Unit'), (sh, None, None), (rd, ns, 'Elaboration'), (sh, None, None), (sh, None, None), (rd, sn, 'Attribution'), (rd, ns, 'Attribution')] for idx, action in enumerate(silver_nuc): sr_parser.operate(action) tree = sr_parser.get_parse_tree() silver_tree = RstTree() silver_tree.assign_tree(tree) silver_tree.assign_doc(gold_tree.doc) silver_tree.back_prop(tree, gold_tree.doc) eval_trees(silver_tree, gold_tree, 1, 9 / 12, 8 / 12) eval_trees(silver_tree, gold_tree, 1, 4 / 6, 5 / 6, use_parseval=True) sr_parser = ParsingState([], []) sr_parser.init(gold_tree.doc) silver_nuc = [(sh, None, None), (sh, None, None), (sh, None, None), (rd, ns, 'Elaboration'), (sh, None, None), (rd, ns, 'Elaboration'), (rd, nn, 'Same-Unit'), (sh, None, None), (rd, ns, 'Elaboration'), (sh, None, None), (sh, None, None), (rd, ns, 'Attribution'), (rd, ns, 'Attribution')] for idx, action in enumerate(silver_nuc): sr_parser.operate(action) tree = sr_parser.get_parse_tree() silver_tree = RstTree() silver_tree.assign_tree(tree) silver_tree.assign_doc(gold_tree.doc) silver_tree.back_prop(tree, gold_tree.doc) eval_trees(silver_tree, gold_tree, 10 / 12, 7 / 12, 5 / 12) eval_trees(silver_tree, gold_tree, 4 / 6, 3 / 6, 3 / 6, use_parseval=True)
def read_rst_trees(data_dir): # Read RST tree file files = [ os.path.join(data_dir, fname) for fname in os.listdir(data_dir) if fname.endswith('.dis') ] rst_trees = [] for fdis in files: fmerge = fdis.replace('.dis', '.merge') if not os.path.isfile(fmerge): raise FileNotFoundError( 'Corresponding .fmerge file does not exist. You should do preprocessing first.' ) rst_tree = RstTree(fdis, fmerge) rst_tree.build() rst_trees.append(rst_tree) return rst_trees
def test_tree(): fdis = "../data/data_dir/train_dir/wsj_0603.out.dis" fmerge = "../data/data_dir/train_dir/wsj_0603.out.merge" tree = RstTree(fdis, fmerge) tree.build() bftbin = RstTree.BFTbin(tree.tree) assert len(bftbin) == 13 edu_spans = [(1, 7), (1, 5), (6, 7), (1, 4), (5, 5), (6, 6), (7, 7), (1, 2), (3, 4), (1, 1), (2, 2), (3, 3), (4, 4)] r, n, s = 'Root', 'Nucleus', 'Satellite' props = [r, n, n, n, s, n, s, n, n, n, s, n, s] rels = [None, 'TextualOrganization', 'TextualOrganization', 'span', 'elaboration-additional', 'span', 'attribution', 'Same-Unit', 'Same-Unit', 'span', 'elaboration-object-attribute-e', 'span', 'elaboration-object-attribute-e',] for i, span in enumerate(bftbin): assert span.edu_span == edu_spans[i] assert span.prop == props[i] assert span.relation == rels[i] # TODO: Check where relation conversion happens? In RST-Coref paper, in generate_action_samples fdis, action_seq = tree.generate_action_samples() assert fdis == fdis sh, rd = 'Shift', 'Reduce' nn, ns, sn = 'NN', 'NS', 'SN' gold_action_seq = [(sh, None), (sh, None), (rd, ns), (sh, None), (sh, None), (rd, ns), (rd, nn), (sh, None), (rd, ns), (sh, None), (sh, None), (rd, ns), (rd, nn)] assert len(action_seq) == len(gold_action_seq) for i in range(len(action_seq)): assert action_seq[i] == gold_action_seq[i] # Check that tree binarization is node correctly fdis = "../data/data_dir/train_dir/file2.dis" fmerge = "../data/data_dir/train_dir/file2.merge" tree = RstTree(fdis, fmerge) tree.build() spans = [span.edu_span for span in RstTree.BFTbin(tree.tree)] nonbinary_spans = [(22, 28), (22, 22), (23, 28), (23, 23), (24, 28), (24, 24), (25, 28), (25, 25), (26, 28), (26, 26), (27, 28), (27, 27), (28, 28), (11, 15), (11, 11), (12, 15), (12, 12), (13, 15), (13, 14), (13, 13), (14, 14), (15, 15), (45, 57), (45, 47), (48, 57), (48, 53), (54, 57), (54, 56), (57, 57)] for span in nonbinary_spans: assert span in spans
def read_rst_trees(data_dir): # Read RST tree file files = [ os.path.join(data_dir, fname) for fname in os.listdir(data_dir) if fname.endswith('.dis') ] for i, fdis in enumerate(files): fmerge = fdis.replace('.dis', '.merge') if not os.path.isfile(fmerge): print( fmerge + " - Corresponding .fmerge file does not exist. Skipping the file." ) continue rst_tree = RstTree(fdis, fmerge) rst_tree.build() if (rst_tree.get_parse().startswith(" ( EDU")): print("Skipping document - tree is corrupt.") continue yield rst_tree
def __getitem__(self, idx): """ Returns: document_edus - packed list of tensors, where each element is tensor of word embeddings for an EDU of that document sorted_idx - output of torch.argsort, used to return EDUs to original linear order y - sequence of gold actions for a particular tree doc - Document instance of an RstTree instance """ doc = Doc() eval_instance = self.X[idx].replace('.dis', '.merge') doc.read_from_fmerge(eval_instance) if self.is_train: return doc, torch.cuda.LongTensor(self.y[idx]) fdis = eval_instance.replace('.merge', '.dis') gold_rst = RstTree(fdis, eval_instance) gold_rst.build() return gold_rst
def eval_parser(self, dev_data=None, path='./examples', save_preds=True, use_parseval=False): """ Test the parsing performance""" # Evaluation met = Metrics(levels=['span', 'nuclearity', 'relation'], use_parseval=use_parseval) # ---------------------------------------- # Read all files from the given path if dev_data is None: dev_data = [ os.path.join(path, fname) for fname in os.listdir(path) if fname.endswith('.dis') ] total_cost = 0 for eval_instance in dev_data: # ---------------------------------------- fmerge = eval_instance.replace('.dis', '.merge') doc = Doc() doc.read_from_fmerge(fmerge) gold_rst = RstTree(eval_instance, fmerge) gold_rst.build() # tok_edus = [nltk.word_tokenize(edu) for edu in doc.doc_edus] tok_edus = [edu.split(" ") for edu in doc.doc_edus] tokens = flatten(tok_edus) coref_document = Document(raw_text=None, tokens=tokens, sents=tok_edus, corefs=[], speakers=["0"] * len(tokens), genre="nw", filename=None) coref_document.token_dict = doc.token_dict coref_document.edu_dict = doc.edu_dict doc = coref_document gold_action_seq, gold_rel_seq = gold_rst.decode_rst_tree() gold_action_seq = [action_map[x] for x in gold_action_seq] gold_relation_seq = [ relation_map[x.lower()] for x in gold_rel_seq if x is not None ] pred_rst, cost = self.parser.sr_parse( doc, torch.cuda.LongTensor(gold_action_seq), torch.cuda.LongTensor(gold_relation_seq)) total_cost += cost if save_preds: if not os.path.isdir('../data/predicted_trees'): os.mkdir('../data/predicted_trees') filename = eval_instance.split(os.sep)[-1] filepath = f'../data/predicted_trees/{self.config[MODEL_NAME]}_{filename}' pred_brackets = pred_rst.bracketing() # Write brackets into file Evaluator.writebrackets(filepath, pred_brackets) # ---------------------------------------- # Evaluate with gold RST tree met.eval(gold_rst, pred_rst) print("Total cost: ", total_cost) if use_parseval: print("Reporting original Parseval metric.") else: print("Reporting RST Parseval metric.") met.report()
def eval_parser(self, path='./examples', report=False, bcvocab=None, draw=True): """ Test the parsing performance""" # Evaluation met = Metrics(levels=['span', 'nuclearity', 'relation']) # ---------------------------------------- # Read all files from the given path doclist = [ os.path.join(path, fname) for fname in os.listdir(path) if fname.endswith('.merge') ] pred_forms = [] gold_forms = [] depth_per_relation = {} for fmerge in doclist: # ---------------------------------------- # Read *.merge file doc = Doc() doc.read_from_fmerge(fmerge) # ---------------------------------------- # Parsing pred_rst = self.parser.sr_parse(doc, bcvocab) if draw: pred_rst.draw_rst(fmerge.replace(".merge", ".ps")) # Get brackets from parsing results pred_brackets = pred_rst.bracketing() fbrackets = fmerge.replace('.merge', '.brackets') # Write brackets into file Evaluator.writebrackets(fbrackets, pred_brackets) # ---------------------------------------- # Evaluate with gold RST tree if report: fdis = fmerge.replace('.merge', '.dis') gold_rst = RstTree(fdis, fmerge) gold_rst.build() met.eval(gold_rst, pred_rst) for node in pred_rst.postorder_DFT(pred_rst.tree, []): pred_forms.append(node.form) for node in gold_rst.postorder_DFT(gold_rst.tree, []): gold_forms.append(node.form) nodes = gold_rst.postorder_DFT(gold_rst.tree, []) inner_nodes = [ node for node in nodes if node.lnode is not None and node.rnode is not None ] for idx, node in enumerate(inner_nodes): relation = node.rnode.relation if node.form == 'NS' else node.lnode.relation rela_class = RstTree.extract_relation(relation) if rela_class in depth_per_relation: depth_per_relation[rela_class].append(node.depth) else: depth_per_relation[rela_class] = [node.depth] lnode_text = ' '.join([ gold_rst.doc.token_dict[tid].word for tid in node.lnode.text ]) lnode_lemmas = ' '.join([ gold_rst.doc.token_dict[tid].lemma for tid in node.lnode.text ]) rnode_text = ' '.join([ gold_rst.doc.token_dict[tid].word for tid in node.rnode.text ]) rnode_lemmas = ' '.join([ gold_rst.doc.token_dict[tid].lemma for tid in node.rnode.text ]) # if rela_class == 'Topic-Change': # print(fmerge) # print(relation) # print(lnode_text) # print(rnode_text) # print() if report: met.report()