Exemplo n.º 1
0
def disconnect_punctuation(trees):
    """
    :param trees: corpus of hybrid trees
    :type trees: __generator[HybridTree]
    :return: corpus of hybrid trees
    :rtype: __generator[GeneralHybridTree]
    lazily disconnect punctuation from each hybrid tree in a corpus of hybrid trees
    """
    for tree in trees:
        tree2 = HybridTree(tree.sent_label())
        for root_id in tree.root:
            if not is_punctuation(tree.node_token(root_id).form()):
                tree2.add_to_root(root_id)
        for id in tree.full_yield():
            token = tree.node_token(id)
            if not is_punctuation(token.form()):
                parent = tree.parent(id)
                while parent and parent not in tree.root and is_punctuation(
                        tree.node_token(parent).form()):
                    parent = tree.parent(parent)
                if parent and is_punctuation(tree.node_token(parent).form()):
                    tree2.add_to_root(id)
                else:
                    tree2.add_child(parent, id)
                tree2.add_node(id, token, True, True)
            else:
                tree2.add_node(id, token, True, False)

        if tree2:
            # basic sanity checks
            if not tree2.root \
                    and len(tree2.id_yield()) == 0 \
                    and len(tree2.nodes()) == len(tree2.full_yield()):
                # Tree consists only of punctuation
                continue
            elif not tree2.root \
                    or tree2.n_nodes() != len(tree2.id_yield()) \
                    or len(tree2.nodes()) != len(tree2.full_yield()):
                print(tree)

                print(tree2)
                print(tree2.sent_label())
                print("Root:", tree2.root)
                print("Nodes: ", tree2.n_nodes())
                print("Id_yield:", len(tree2.id_yield()), tree2.id_yield())
                print("Nodes: ", len(tree2.nodes()))
                print("full yield: ", len(tree2.full_yield()))
                raise Exception()
            yield tree2
Exemplo n.º 2
0
def parse_conll_corpus(path, ignore_punctuation, limit=sys.maxsize, start=0):
    """
    :param path: path to corpus
    :type: str
    :param ignore_punctuation: exclude punctuation from tree structure
    :type ignore_punctuation: bool
    :param limit: stop generation after limit trees
    :type: int
    :param start: start generation with start'th tree
    :type start: int
    :return: a series of hybrid trees read from file
    :rtype: __generator[HybridTree]
    :raise Exception: unexpected input in corpus file
    Lazily parses a dependency corpus (in CoNLL format) and generates GeneralHybridTrees.
    """

    # print path
    with open(path) as file_content:
        tree_count = 0

        while tree_count < limit:
            tree = None

            try:
                line = next(file_content)
                while line.startswith('#'):
                    line = next(file_content)
            except StopIteration:
                break

            match = CONLL_LINE.match(line)
            while match:
                if match.group(1) == '1':
                    tree_count += 1
                    tree = HybridTree('tree' + str(tree_count))

                node_id = match.group(1)
                form = match.group(2)
                lemma = match.group(3)
                cpos = match.group(4)
                pos = match.group(5)
                feats = match.group(6)
                parent = match.group(7)
                deprel = match.group(8)

                # We ignore information about multiple token's as present in the UD version of Prague Dep. TB
                if MULTI_TOKEN.search(node_id):
                  pass
                else:
                    # If punctuation is to be ignored, we
                    # remove it from the hybrid tree
                    # Punctuation according to definition
                    # cf. http://ilk.uvt.nl/conll/software.html#eval

                    # if not ignore_punctuation or form.translate(no_translation, string.punctuation):
                    tree.add_node(node_id, CoNLLToken(form, lemma, cpos, pos, feats, deprel), True, True)
                    if parent != '0':
                        tree.add_child(parent, node_id)
                    # else:
                    #    tree.add_node(node_id, CoNLLToken(form, lemma, pos, fine_grained_pos, feats, deprel), True, False)

                    # TODO: If punctuation is ignored and the root is punctuation,
                    # TODO: it is added to the tree anyhow.
                    if parent == '0':
                        tree.add_to_root(node_id)

                try:
                    line = next(file_content)
                    while line.startswith('#'):
                        line = next(file_content)
                    match = CONLL_LINE.search(line)
                except StopIteration:
                    line = ''
                    match = None

            # Assume empty line, otherwise raise exception
            match = EMPTY_LINE.match(line)
            if not match:
                raise Exception("Unexpected input in CoNLL corpus file.")

            if tree:
                # basic sanity checks
                if not tree.root:
                    # FIXME: ignoring punctuation may leads to malformed trees
                    print("non-rooted")
                    if ignore_punctuation:
                        continue
                    raise Exception
                    # elif root > 1:
                    # FIXME: turkish corpus contains trees with more than one root
                    # FIXME: currently, they are ignored
                    # continue
                elif tree.n_nodes() != len(tree.id_yield()) or len(tree.nodes()) != len(tree.full_yield()):
                    # FIXME: ignoring punctuation may leads to malformed trees
                    if ignore_punctuation:
                        continue
                    raise Exception(
                        '{4}: connected nodes: {0}, total nodes: {1}, full yield: {2}, connected yield: {3}'.format(
                            str(tree.n_nodes()), str(len(tree.nodes())), str(len(tree.full_yield())),
                            str(len(tree.id_yield())), tree.sent_label()))
                if tree_count > start:
                    yield tree
class GeneralHybridTreeTestCase(unittest.TestCase):
    tree = None

    def setUp(self):
        self.tree = HybridTree()
        self.tree.add_node("v1", construct_conll_token("Piet", "NP"), True)
        self.tree.add_node("v21", construct_conll_token("Marie", "N"), True)
        self.tree.add_node("v", construct_conll_token("helpen", "VP"), True)
        self.tree.add_node("v2", construct_conll_token("lezen", "V"), True)
        self.tree.add_child("v", "v2")
        self.tree.add_child("v", "v1")
        self.tree.add_child("v2", "v21")
        self.tree.add_node("v3", construct_conll_token(".", "Punc"), True,
                           False)
        self.tree.add_to_root("v")

    def test_children(self):
        self.assertListEqual(self.tree.children('v'), ['v2', 'v1'])
        self.tree.reorder()
        self.assertListEqual(self.tree.children('v'), ['v1', 'v2'])

    def test_fringe(self):
        self.tree.reorder()
        self.assertListEqual(self.tree.fringe('v'), [2, 0, 3, 1])
        self.assertListEqual(self.tree.fringe('v2'), [3, 1])

    def test_n_spans(self):
        self.tree.reorder()
        self.assertEqual(self.tree.n_spans('v'), 1)
        self.assertEqual(self.tree.n_spans('v2'), 2)

    def test_n_gaps(self):
        self.tree.reorder()
        self.assertEqual(self.tree.n_gaps(), 1)

    def test_node_ids(self):
        self.tree.reorder()
        self.assertListEqual(sorted(self.tree.nodes()),
                             sorted(['v', 'v1', 'v2', 'v21', 'v3']))

    def test_complete(self):
        self.tree.reorder()
        self.assertEqual(self.tree.complete(), True)

    def test_unlabelled_structure(self):
        self.tree.reorder()
        self.assertTupleEqual(self.tree.unlabelled_structure(),
                              ({0, 1, 2, 3}, [({0}, []),
                                              ({1, 3}, [({1}, [])])]))

    def test_max_n_spans(self):
        self.tree.reorder()
        self.assertEqual(self.tree.max_n_spans(), 2)

    def test_labelled_yield(self):
        self.tree.reorder()
        self.assertListEqual(
            [token.form() for token in self.tree.token_yield()],
            "Piet Marie helpen lezen".split(' '))

    def test_full_labelled_yield(self):
        self.tree.reorder()
        self.assertListEqual(
            [token.form() for token in self.tree.full_token_yield()],
            "Piet Marie helpen lezen .".split(' '))

    def test_full_yield(self):
        self.tree.reorder()
        self.assertListEqual(self.tree.full_yield(),
                             'v1 v21 v v2 v3'.split(' '))

    # def test_labelled_spans(self):
    # self.tree.reorder()
    # self.assertListEqual(self.tree.labelled_spans(), [])

    def test_pos_yield(self):
        self.tree.reorder()
        self.assertListEqual(
            [token.pos() for token in self.tree.token_yield()],
            "NP N VP V".split(' '))

    def test_recursive_partitioning(self):
        self.tree.reorder()
        self.assertEqual(self.tree.recursive_partitioning(),
                         ({0, 1, 2, 3}, [({0}, []),
                                         ({1, 3}, [({1}, []), ({3}, [])]),
                                         ({2}, [])]))