Esempio n. 1
0
 def test_split_long_raises_on_empty(self):
     with self.assertRaises(ValueError):
         unified_tokenizer.split_long_token(token_string='',
                                            max_output_token_length=3)
Esempio n. 2
0
 def test_split_long_returns_expected(self, token, components):
     actual_components = unified_tokenizer.split_long_token(
         token_string=token, max_output_token_length=3)
     self.assertSequenceEqual(components, actual_components)