def test_simple(self): data = dedent("""\ # global.columns = ID FORM UPOS HEAD DEPREL MISC PARSEME:MWE 1\tDer\tDET\t2\tdet\t_\t* """) self.assertEqual( parse_conllu_plus_fields(string_to_file(data)), ["id", "form", "upos", "head", "deprel", "misc", "parseme:mwe"])
def test_simple(self): data = dedent("""\ 1\thej 2\tdå 3\thej 1\thej 2\tdå 3\thej """) sentences = list(parse_sentences(string_to_file(data))) self.assertEqual(sentences, [ '1\thej\n2\tdå\n3\thej', '1\thej\n2\tdå\n3\thej', ])
def test_multiple_newlines(self): data = dedent("""\ 1\thej 2\tdå 1\thej 2\tdå 1\thej 2\tdå """) sentences = list(parse_sentences(string_to_file(data))) self.assertEqual(sentences, [ '1\thej\n2\tdå', '1\thej\n2\tdå', '1\thej\n2\tdå', ])
def parse(data, fields=None, field_parsers=None, metadata_parsers=None): return list( parse_incr(string_to_file(data), fields=fields, field_parsers=field_parsers, metadata_parsers=metadata_parsers))
def parse_tree(data): return list(parse_tree_incr(string_to_file(data)))
def test_ends_without_newline(self): data = "1\thej\n2\tdå" sentences = list(parse_sentences(string_to_file(data))) self.assertEqual(sentences, [ '1\thej\n2\tdå', ])
def test_empty(self): self.assertEqual(list(parse_sentences(string_to_file(""))), []) self.assertEqual(list(parse_sentences(string_to_file(None))), [])
def test_empty_columns(self): data = dedent("""\ # global.columns = 1\tDer\tDET\t2\tdet\t_\t* """) self.assertEqual(parse_conllu_plus_fields(string_to_file(data)), None)
def test_empty(self): self.assertEqual(parse_conllu_plus_fields(string_to_file("")), None) self.assertEqual(parse_conllu_plus_fields(string_to_file(None)), None)
def test_parse_tree_incr(self): self.assertEqual(parse_tree(data), list(parse_tree_incr(string_to_file(data))))