Python WordTokenizer.tokenize_sign Beispiele

Beispiel #1

 def test_akkadian_sign_tokenizer(self):
     """
     Tests sign_tokenizer.
     """
     tokenizer = WordTokenizer('akkadian')
     word = ("{gisz}isz-pur-ram", "akkadian")
     output = tokenizer.tokenize_sign(word)
     goal = [("gisz", "determinative"), ("isz", "akkadian"),
             ("pur", "akkadian"), ("ram", "akkadian")]
     self.assertEqual(output, goal)

Beispiel #2

Datei: test_tokenize.py Projekt: cltk/cltk

 def test_akkadian_sign_tokenizer(self):
     """
     Tests sign_tokenizer.
     """
     tokenizer = WordTokenizer('akkadian')
     word = ("{gisz}isz-pur-ram", "akkadian")
     output = tokenizer.tokenize_sign(word)
     goal = [("gisz", "determinative"), ("isz", "akkadian"),
             ("pur", "akkadian"), ("ram", "akkadian")]
     self.assertEqual(output, goal)