def test_simple(self):
data = dedent("""\
# global.columns = ID FORM UPOS HEAD DEPREL MISC PARSEME:MWE
1\tDer\tDET\t2\tdet\t_\t*
""")
self.assertEqual(
parse_conllu_plus_fields(string_to_file(data)),
["id", "form", "upos", "head", "deprel", "misc", "parseme:mwe"])
def test_simple(self):
data = dedent("""\
1\thej
2\tdå
3\thej
1\thej
2\tdå
3\thej
""")
sentences = list(parse_sentences(string_to_file(data)))
self.assertEqual(sentences, [
'1\thej\n2\tdå\n3\thej',
'1\thej\n2\tdå\n3\thej',
])
def test_multiple_newlines(self):
data = dedent("""\
1\thej
2\tdå
1\thej
2\tdå
1\thej
2\tdå
""")
sentences = list(parse_sentences(string_to_file(data)))
self.assertEqual(sentences, [
'1\thej\n2\tdå',
'1\thej\n2\tdå',
'1\thej\n2\tdå',
])
def parse(data, fields=None, field_parsers=None, metadata_parsers=None):
return list(
parse_incr(string_to_file(data),
fields=fields,
field_parsers=field_parsers,
metadata_parsers=metadata_parsers))
def parse_tree(data):
return list(parse_tree_incr(string_to_file(data)))
def test_ends_without_newline(self):
data = "1\thej\n2\tdå"
sentences = list(parse_sentences(string_to_file(data)))
self.assertEqual(sentences, [
'1\thej\n2\tdå',
])
def test_empty(self):
self.assertEqual(list(parse_sentences(string_to_file(""))), [])
self.assertEqual(list(parse_sentences(string_to_file(None))), [])
def test_empty_columns(self):
data = dedent("""\
# global.columns =
1\tDer\tDET\t2\tdet\t_\t*
""")
self.assertEqual(parse_conllu_plus_fields(string_to_file(data)), None)
def test_empty(self):
self.assertEqual(parse_conllu_plus_fields(string_to_file("")), None)
self.assertEqual(parse_conllu_plus_fields(string_to_file(None)), None)
def test_parse_tree_incr(self):
self.assertEqual(parse_tree(data), list(parse_tree_incr(string_to_file(data))))
