def disconnect_punctuation(trees): """ :param trees: corpus of hybrid trees :type trees: __generator[HybridTree] :return: corpus of hybrid trees :rtype: __generator[GeneralHybridTree] lazily disconnect punctuation from each hybrid tree in a corpus of hybrid trees """ for tree in trees: tree2 = HybridTree(tree.sent_label()) for root_id in tree.root: if not is_punctuation(tree.node_token(root_id).form()): tree2.add_to_root(root_id) for id in tree.full_yield(): token = tree.node_token(id) if not is_punctuation(token.form()): parent = tree.parent(id) while parent and parent not in tree.root and is_punctuation( tree.node_token(parent).form()): parent = tree.parent(parent) if parent and is_punctuation(tree.node_token(parent).form()): tree2.add_to_root(id) else: tree2.add_child(parent, id) tree2.add_node(id, token, True, True) else: tree2.add_node(id, token, True, False) if tree2: # basic sanity checks if not tree2.root \ and len(tree2.id_yield()) == 0 \ and len(tree2.nodes()) == len(tree2.full_yield()): # Tree consists only of punctuation continue elif not tree2.root \ or tree2.n_nodes() != len(tree2.id_yield()) \ or len(tree2.nodes()) != len(tree2.full_yield()): print(tree) print(tree2) print(tree2.sent_label()) print("Root:", tree2.root) print("Nodes: ", tree2.n_nodes()) print("Id_yield:", len(tree2.id_yield()), tree2.id_yield()) print("Nodes: ", len(tree2.nodes())) print("full yield: ", len(tree2.full_yield())) raise Exception() yield tree2
def parse_conll_corpus(path, ignore_punctuation, limit=sys.maxsize, start=0): """ :param path: path to corpus :type: str :param ignore_punctuation: exclude punctuation from tree structure :type ignore_punctuation: bool :param limit: stop generation after limit trees :type: int :param start: start generation with start'th tree :type start: int :return: a series of hybrid trees read from file :rtype: __generator[HybridTree] :raise Exception: unexpected input in corpus file Lazily parses a dependency corpus (in CoNLL format) and generates GeneralHybridTrees. """ # print path with open(path) as file_content: tree_count = 0 while tree_count < limit: tree = None try: line = next(file_content) while line.startswith('#'): line = next(file_content) except StopIteration: break match = CONLL_LINE.match(line) while match: if match.group(1) == '1': tree_count += 1 tree = HybridTree('tree' + str(tree_count)) node_id = match.group(1) form = match.group(2) lemma = match.group(3) cpos = match.group(4) pos = match.group(5) feats = match.group(6) parent = match.group(7) deprel = match.group(8) # We ignore information about multiple token's as present in the UD version of Prague Dep. TB if MULTI_TOKEN.search(node_id): pass else: # If punctuation is to be ignored, we # remove it from the hybrid tree # Punctuation according to definition # cf. http://ilk.uvt.nl/conll/software.html#eval # if not ignore_punctuation or form.translate(no_translation, string.punctuation): tree.add_node(node_id, CoNLLToken(form, lemma, cpos, pos, feats, deprel), True, True) if parent != '0': tree.add_child(parent, node_id) # else: # tree.add_node(node_id, CoNLLToken(form, lemma, pos, fine_grained_pos, feats, deprel), True, False) # TODO: If punctuation is ignored and the root is punctuation, # TODO: it is added to the tree anyhow. if parent == '0': tree.add_to_root(node_id) try: line = next(file_content) while line.startswith('#'): line = next(file_content) match = CONLL_LINE.search(line) except StopIteration: line = '' match = None # Assume empty line, otherwise raise exception match = EMPTY_LINE.match(line) if not match: raise Exception("Unexpected input in CoNLL corpus file.") if tree: # basic sanity checks if not tree.root: # FIXME: ignoring punctuation may leads to malformed trees print("non-rooted") if ignore_punctuation: continue raise Exception # elif root > 1: # FIXME: turkish corpus contains trees with more than one root # FIXME: currently, they are ignored # continue elif tree.n_nodes() != len(tree.id_yield()) or len(tree.nodes()) != len(tree.full_yield()): # FIXME: ignoring punctuation may leads to malformed trees if ignore_punctuation: continue raise Exception( '{4}: connected nodes: {0}, total nodes: {1}, full yield: {2}, connected yield: {3}'.format( str(tree.n_nodes()), str(len(tree.nodes())), str(len(tree.full_yield())), str(len(tree.id_yield())), tree.sent_label())) if tree_count > start: yield tree
def n_nodes(self): return HybridTree.n_nodes(self) + 1