def test_load_tokenizer_no_class(self): config = BrainConfiguration() self.assertIsNotNone(config) tokenizer_config = BrainTokenizerConfiguration() self.assertIsNotNone(tokenizer_config) tokenizer_config._classname = None tokenizer_config._punctation_chars = '' config._tokenizer = tokenizer_config tokenizer = Tokenizer.load_tokenizer(config) self.assertIsNotNone(tokenizer) self.assertIsInstance(tokenizer, Tokenizer)
def test_load_tokenizer(self): config = BrainConfiguration() self.assertIsNotNone(config) tokenizer_config = BrainTokenizerConfiguration() self.assertIsNotNone(tokenizer_config) tokenizer_config._classname = 'programy.dialog.tokenizer.tokenizer.Tokenizer' tokenizer_config._punctation_chars = '' config._tokenizer = tokenizer_config tokenizer = Tokenizer.load_tokenizer(config) self.assertIsNotNone(tokenizer) self.assertIsInstance(tokenizer, Tokenizer)