def test_metadata(self):
data = dedent("""\
# data = meta
# meta = data
1\tdog
""")
tokenlist = TokenList(*parse_token_and_metadata(data))
self.assertEqual(serialize(tokenlist), data)
def serialize(self):
if not self.token or "id" not in self.token:
raise ParseException("Could not serialize tree, missing 'id' field.")
def flatten_tree(root_token, token_list=[]):
token_list.append(root_token.token)
for child_token in root_token.children:
flatten_tree(child_token, token_list)
return token_list
tokens = flatten_tree(self)
tokens = sorted(tokens, key=lambda t: t['id'])
tokenlist = TokenList(tokens, self.metadata)
return serialize(tokenlist)
def test_serialize_tricky_fields(self):
data = dedent("""\
5\tjumps\tjump\tVERB\tVBZ\tMood=Ind|Number=Sing\t0\troot\t_\tSpaceAfter=No
""")
tokenlist = TokenList(*parse_token_and_metadata(data))
self.assertEqual(serialize(tokenlist).strip(), data.strip())
def test_identity_unicode(self):
data = "5\tlängtar\n\n"
tokenlist = TokenList(*parse_token_and_metadata(data))
self.assertEqual(serialize(tokenlist), data)
def serialize(self):
return serialize(self)
def test_serialize_on_tokenlist(self):
tokenlist = TokenList([{"id": 1}])
self.assertEqual(tokenlist.serialize(), serialize(tokenlist))
