예제 #1
0
파일: tests.py 프로젝트: eipiplusun/educe
    def test_rst_to_dt(self):
        lw_trees = ["(R:rel (S x) (N y))",

                    """
                    (R:rel
                        (S x)
                        (N:rel (N h) (S t)))
                    """,

                    """
                    (R:r
                        (S x)
                        (N:r (N:r (S t1) (N h))
                             (S t2)))
                    """
        ]

        for lstr in lw_trees:
            rst1 = parse_lightweight_tree(lstr)
            dep = RstDepTree.from_simple_rst_tree(rst1)
            rst2 = deptree_to_simple_rst_tree(dep)
            self.assertEqual(rst1, rst2, "round-trip on " + lstr)

        for name, tree in self._test_trees().items():
            rst1 = SimpleRSTTree.from_rst_tree(tree)
            dep = RstDepTree.from_simple_rst_tree(rst1)
            rst2 = deptree_to_simple_rst_tree(dep)
            self.assertEqual(treenode(rst1).span,
                             treenode(rst2).span,
                             "span equality on " + name)
            self.assertEqual(treenode(rst1).edu_span,
                             treenode(rst2).edu_span,
                             "edu span equality on " + name)
예제 #2
0
    def test_rst_to_dt(self):
        lw_trees = [
            "(R:rel (S x) (N y))", """
                    (R:rel
                        (S x)
                        (N:rel (N h) (S t)))
                    """, """
                    (R:r
                        (S x)
                        (N:r (N:r (S t1) (N h))
                             (S t2)))
                    """
        ]

        for lstr in lw_trees:
            rst1 = parse_lightweight_tree(lstr)
            dep = RstDepTree.from_simple_rst_tree(rst1)
            rst2 = deptree_to_simple_rst_tree(dep)
            self.assertEqual(rst1, rst2, "round-trip on " + lstr)

        for name, tree in self._test_trees().items():
            rst1 = SimpleRSTTree.from_rst_tree(tree)
            dep = RstDepTree.from_simple_rst_tree(rst1)
            rst2 = deptree_to_simple_rst_tree(dep)
            self.assertEqual(
                treenode(rst1).span,
                treenode(rst2).span, "span equality on " + name)
            self.assertEqual(
                treenode(rst1).edu_span,
                treenode(rst2).edu_span, "edu span equality on " + name)
예제 #3
0
파일: deptree.py 프로젝트: eipiplusun/educe
def convert(corpus, multinuclear, odir):
    """
    Convert every RST tree in the corpus to a dependency tree
    (and back, but simplified using a set of relation types
    that will be systematically treated as multinuclear)
    """
    bin_dir = os.path.join(odir, "rst-binarised")
    dt_dir = os.path.join(odir, "rst-to-dt")
    rst2_dir = os.path.join(odir, "dt-to-rst")
    for subdir in [bin_dir, dt_dir, rst2_dir]:
        if not os.path.exists(subdir):
            os.makedirs(subdir)

    for k in corpus:
        suffix = os.path.splitext(k.doc)[0]

        stree = educe.rst_dt.SimpleRSTTree.from_rst_tree(corpus[k])
        with open(os.path.join(bin_dir, suffix), 'w') as fout:
            fout.write(str(stree))

        dtree = RstDepTree.from_simple_rst_tree(stree)
        with open(os.path.join(dt_dir, suffix), 'w') as fout:
            fout.write(str(dtree))

        stree2 = deptree_to_simple_rst_tree(dtree, multinuclear)
        with open(os.path.join(rst2_dir, suffix), 'w') as fout:
            fout.write(str(stree2))
예제 #4
0
def convert(corpus, multinuclear, odir):
    """
    Convert every RST tree in the corpus to a dependency tree
    (and back, but simplified using a set of relation types
    that will be systematically treated as multinuclear)
    """
    bin_dir = os.path.join(odir, "rst-binarised")
    dt_dir = os.path.join(odir, "rst-to-dt")
    rst2_dir = os.path.join(odir, "dt-to-rst")
    for subdir in [bin_dir, dt_dir, rst2_dir]:
        if not os.path.exists(subdir):
            os.makedirs(subdir)

    for k in corpus:
        suffix = os.path.splitext(k.doc)[0]

        stree = SimpleRSTTree.from_rst_tree(corpus[k])
        with open(os.path.join(bin_dir, suffix), 'w') as fout:
            fout.write(str(stree))

        dtree = RstDepTree.from_simple_rst_tree(stree)
        with open(os.path.join(dt_dir, suffix), 'w') as fout:
            fout.write(str(dtree))

        stree2 = deptree_to_simple_rst_tree(dtree)
        with open(os.path.join(rst2_dir, suffix), 'w') as fout:
            fout.write(str(stree2))
예제 #5
0
    def test_rst_to_dt_nuclearity_loss(self):
        """
        Test that we still get sane tree structure with
        nuclearity loss
        """
        tricky = """
                 (R:r (S t) (N h))
                 """

        nuked = """
                (R:r (N t) (N h))
                """

        #        tricky = """
        #                 (R:r
        #                     (S x)
        #                     (N:r (N:r (S t1) (N h))
        #                          (S t2)))
        #                 """
        #
        #        nuked = """
        #                 (R:r
        #                     (N x)
        #                     (N:r (N:r (N t1) (N h))
        #                          (N t2)))
        #                 """

        rst0 = parse_lightweight_tree(nuked)
        rst1 = parse_lightweight_tree(tricky)

        # a little sanity check first
        dep0 = RstDepTree.from_simple_rst_tree(rst0)
        rev0 = deptree_to_simple_rst_tree(dep0)  # was:, ['r'])
        self.assertEqual(rst0, rev0, "same structure " + nuked)  # sanity

        # now the real test
        dep1 = RstDepTree.from_simple_rst_tree(rst1)
        rev1 = deptree_to_simple_rst_tree(dep1)  # was:, ['r'])
예제 #6
0
파일: tests.py 프로젝트: eipiplusun/educe
    def test_rst_to_dt_nuclearity_loss(self):
        """
        Test that we still get sane tree structure with
        nuclearity loss
        """
        tricky = """
                 (R:r (S t) (N h))
                 """

        nuked = """
                (R:r (N t) (N h))
                """

#        tricky = """
#                 (R:r
#                     (S x)
#                     (N:r (N:r (S t1) (N h))
#                          (S t2)))
#                 """
#
#        nuked = """
#                 (R:r
#                     (N x)
#                     (N:r (N:r (N t1) (N h))
#                          (N t2)))
#                 """

        rst0 = parse_lightweight_tree(nuked)
        rst1 = parse_lightweight_tree(tricky)

        # a little sanity check first
        dep0 = RstDepTree.from_simple_rst_tree(rst0)
        rev0 = deptree_to_simple_rst_tree(dep0)  # was:, ['r'])
        self.assertEqual(rst0, rev0, "same structure " + nuked)  # sanity

        # now the real test
        dep1 = RstDepTree.from_simple_rst_tree(rst1)
        rev1 = deptree_to_simple_rst_tree(dep1)  # was:, ['r'])
예제 #7
0
    def test_dt_to_rst_order(self):
        lw_trees = [
            "(R:r (N:r (N h) (S r1)) (S r2))",
            "(R:r (S:r (S l2) (N l1)) (N h))",
            "(R:r (N:r (S l1) (N h)) (S r1))",
            """
            (R:r
              (N:r
                (N:r (S l2)
                     (N:r (S l1)
                          (N h)))
                (S r1))
              (S r2))
            """,  # ((l2 <- l1 <- h) -> r1 -> r2)
            """
            (R:r
              (N:r
                (S l2)
                (N:r (N:r (S l1)
                          (N h))
                     (S r1)))
              (S r2))
            """,  # (l2 <- ((l1 <- h) -> r1)) -> r2
        ]

        for lstr in lw_trees:
            rst1 = parse_lightweight_tree(lstr)
            dep = RstDepTree.from_simple_rst_tree(rst1)

            dep_a = dep
            rst2a = deptree_to_simple_rst_tree(dep_a)
            self.assertEqual(rst1, rst2a, "round-trip on " + lstr)

            dep_b = copy.deepcopy(dep)
            dep_b.deps(0).reverse()
            rst2b = deptree_to_simple_rst_tree(dep_b)
            # TODO assertion on rst2b?

            dep_c = copy.deepcopy(dep)
            random.shuffle(dep_c.deps(0))
            rst2c = deptree_to_simple_rst_tree(dep_c)
예제 #8
0
        'corpus': os.path.relpath(rst_corpus_dir, start=DATA_DIR),
        'strip_accents': strip_accents,
        'lowercase': lowercase,
        'stop_words': stop_words,
        'n_jobs': n_jobs,
        'verbose': verbose,
    }
    print('# parameters: ({})'.format(params),
          file=outfile)

    # do the real job
    corpus_items = sorted(rst_corpus.items())
    doc_keys = [key.doc for key, doc in corpus_items]
    doc_key_dtrees = [
        (doc_key.doc,
         RstDepTree.from_simple_rst_tree(SimpleRSTTree.from_rst_tree(doc)))
        for doc_key, doc in corpus_items
    ]
    edu_txts = list(e.text().replace('\n', ' ')
                    for doc_key, dtree in doc_key_dtrees
                    for e in dtree.edus)
    # vectorize each EDU using its text
    edu_vecs = vect.transform(edu_txts)
    # normalize each row of the count matrix using the l1 norm
    # (copy=False to perform in place)
    edu_vecs = normalize(edu_vecs, norm='l1', copy=False)
    # get all pairs of EDUs of interest, here as triples
    # (gov_idx, dep_idx, lbl)
    # TODO maybe sort edu pairs so that dependents with
    # the same governor are grouped (potential speed up?)
    edu_pairs = [
예제 #9
0
        'corpus': os.path.relpath(rst_corpus_dir, start=DATA_DIR),
        'strip_accents': strip_accents,
        'lowercase': lowercase,
        'stop_words': stop_words,
        'n_jobs': n_jobs,
        'verbose': verbose,
    }
    print('# parameters: ({})'.format(params),
          file=outfile)

    # do the real job
    corpus_items = sorted(rst_corpus.items())
    doc_keys = [key.doc for key, doc in corpus_items]
    doc_key_dtrees = [
        (doc_key.doc,
         RstDepTree.from_simple_rst_tree(SimpleRSTTree.from_rst_tree(doc)))
        for doc_key, doc in corpus_items
    ]
    edu_txts = list(e.text().replace('\n', ' ')
                    for doc_key, dtree in doc_key_dtrees
                    for e in dtree.edus)
    # vectorize each EDU using its text
    edu_vecs = vect.transform(edu_txts)
    # normalize each row of the count matrix using the l1 norm
    # (copy=False to perform in place)
    edu_vecs = normalize(edu_vecs, norm='l1', copy=False)
    # get all pairs of EDUs of interest, here as triples
    # (gov_idx, dep_idx, lbl)
    # TODO maybe sort edu pairs so that dependents with
    # the same governor are grouped (potential speed up?)
    edu_pairs = [