def test_metadata(self): data = dedent("""\ # data = meta # meta = data 1\tdog """) tokenlist = TokenList(*parse_token_and_metadata(data)) self.assertEqual(serialize(tokenlist), data)
def serialize(self): if not self.token or "id" not in self.token: raise ParseException("Could not serialize tree, missing 'id' field.") def flatten_tree(root_token, token_list=[]): token_list.append(root_token.token) for child_token in root_token.children: flatten_tree(child_token, token_list) return token_list tokens = flatten_tree(self) tokens = sorted(tokens, key=lambda t: t['id']) tokenlist = TokenList(tokens, self.metadata) return serialize(tokenlist)
def test_serialize_tricky_fields(self): data = dedent("""\ 5\tjumps\tjump\tVERB\tVBZ\tMood=Ind|Number=Sing\t0\troot\t_\tSpaceAfter=No """) tokenlist = TokenList(*parse_token_and_metadata(data)) self.assertEqual(serialize(tokenlist).strip(), data.strip())
def test_identity_unicode(self): data = "5\tlängtar\n\n" tokenlist = TokenList(*parse_token_and_metadata(data)) self.assertEqual(serialize(tokenlist), data)
def serialize(self): return serialize(self)
def test_serialize_on_tokenlist(self): tokenlist = TokenList([{"id": 1}]) self.assertEqual(tokenlist.serialize(), serialize(tokenlist))