def disconnect_punctuation(trees):
"""
:param trees: corpus of hybrid trees
:type trees: __generator[HybridTree]
:return: corpus of hybrid trees
:rtype: __generator[GeneralHybridTree]
lazily disconnect punctuation from each hybrid tree in a corpus of hybrid trees
"""
for tree in trees:
tree2 = HybridTree(tree.sent_label())
for root_id in tree.root:
if not is_punctuation(tree.node_token(root_id).form()):
tree2.add_to_root(root_id)
for id in tree.full_yield():
token = tree.node_token(id)
if not is_punctuation(token.form()):
parent = tree.parent(id)
while parent and parent not in tree.root and is_punctuation(
tree.node_token(parent).form()):
parent = tree.parent(parent)
if parent and is_punctuation(tree.node_token(parent).form()):
tree2.add_to_root(id)
else:
tree2.add_child(parent, id)
tree2.add_node(id, token, True, True)
else:
tree2.add_node(id, token, True, False)
if tree2:
# basic sanity checks
if not tree2.root \
and len(tree2.id_yield()) == 0 \
and len(tree2.nodes()) == len(tree2.full_yield()):
# Tree consists only of punctuation
continue
elif not tree2.root \
or tree2.n_nodes() != len(tree2.id_yield()) \
or len(tree2.nodes()) != len(tree2.full_yield()):
print(tree)
print(tree2)
print(tree2.sent_label())
print("Root:", tree2.root)
print("Nodes: ", tree2.n_nodes())
print("Id_yield:", len(tree2.id_yield()), tree2.id_yield())
print("Nodes: ", len(tree2.nodes()))
print("full yield: ", len(tree2.full_yield()))
raise Exception()
yield tree2
def parse_conll_corpus(path, ignore_punctuation, limit=sys.maxsize, start=0):
"""
:param path: path to corpus
:type: str
:param ignore_punctuation: exclude punctuation from tree structure
:type ignore_punctuation: bool
:param limit: stop generation after limit trees
:type: int
:param start: start generation with start'th tree
:type start: int
:return: a series of hybrid trees read from file
:rtype: __generator[HybridTree]
:raise Exception: unexpected input in corpus file
Lazily parses a dependency corpus (in CoNLL format) and generates GeneralHybridTrees.
"""
# print path
with open(path) as file_content:
tree_count = 0
while tree_count < limit:
tree = None
try:
line = next(file_content)
while line.startswith('#'):
line = next(file_content)
except StopIteration:
break
match = CONLL_LINE.match(line)
while match:
if match.group(1) == '1':
tree_count += 1
tree = HybridTree('tree' + str(tree_count))
node_id = match.group(1)
form = match.group(2)
lemma = match.group(3)
cpos = match.group(4)
pos = match.group(5)
feats = match.group(6)
parent = match.group(7)
deprel = match.group(8)
# We ignore information about multiple token's as present in the UD version of Prague Dep. TB
if MULTI_TOKEN.search(node_id):
pass
else:
# If punctuation is to be ignored, we
# remove it from the hybrid tree
# Punctuation according to definition
# cf. http://ilk.uvt.nl/conll/software.html#eval
# if not ignore_punctuation or form.translate(no_translation, string.punctuation):
tree.add_node(node_id, CoNLLToken(form, lemma, cpos, pos, feats, deprel), True, True)
if parent != '0':
tree.add_child(parent, node_id)
# else:
# tree.add_node(node_id, CoNLLToken(form, lemma, pos, fine_grained_pos, feats, deprel), True, False)
# TODO: If punctuation is ignored and the root is punctuation,
# TODO: it is added to the tree anyhow.
if parent == '0':
tree.add_to_root(node_id)
try:
line = next(file_content)
while line.startswith('#'):
line = next(file_content)
match = CONLL_LINE.search(line)
except StopIteration:
line = ''
match = None
# Assume empty line, otherwise raise exception
match = EMPTY_LINE.match(line)
if not match:
raise Exception("Unexpected input in CoNLL corpus file.")
if tree:
# basic sanity checks
if not tree.root:
# FIXME: ignoring punctuation may leads to malformed trees
print("non-rooted")
if ignore_punctuation:
continue
raise Exception
# elif root > 1:
# FIXME: turkish corpus contains trees with more than one root
# FIXME: currently, they are ignored
# continue
elif tree.n_nodes() != len(tree.id_yield()) or len(tree.nodes()) != len(tree.full_yield()):
# FIXME: ignoring punctuation may leads to malformed trees
if ignore_punctuation:
continue
raise Exception(
'{4}: connected nodes: {0}, total nodes: {1}, full yield: {2}, connected yield: {3}'.format(
str(tree.n_nodes()), str(len(tree.nodes())), str(len(tree.full_yield())),
str(len(tree.id_yield())), tree.sent_label()))
if tree_count > start:
yield tree