def test_load_tokenizer_no_class(self):
config = BrainConfiguration()
self.assertIsNotNone(config)
tokenizer_config = BrainTokenizerConfiguration()
self.assertIsNotNone(tokenizer_config)
tokenizer_config._classname = None
tokenizer_config._punctation_chars = ''
config._tokenizer = tokenizer_config
tokenizer = Tokenizer.load_tokenizer(config)
self.assertIsNotNone(tokenizer)
self.assertIsInstance(tokenizer, Tokenizer)
def test_load_tokenizer(self):
config = BrainConfiguration()
self.assertIsNotNone(config)
tokenizer_config = BrainTokenizerConfiguration()
self.assertIsNotNone(tokenizer_config)
tokenizer_config._classname = 'programy.dialog.tokenizer.tokenizer.Tokenizer'
tokenizer_config._punctation_chars = ''
config._tokenizer = tokenizer_config
tokenizer = Tokenizer.load_tokenizer(config)
self.assertIsNotNone(tokenizer)
self.assertIsInstance(tokenizer, Tokenizer)
def test_load_tokenizer_classname_none(self):
config = BrainTokenizerConfiguration()
config._classname = None
tokenizer = Tokenizer.load_tokenizer(config)
self.assertIsNotNone(tokenizer)
self.assertIsInstance(tokenizer, Tokenizer)
def test_load_invalid_tokenizer(self):
config = BrainTokenizerConfiguration()
config._classname = "programy.dialog.tokenizer.tokenizer.TokenizerXXX"
tokenizer = Tokenizer.load_tokenizer(config)
self.assertIsNotNone(tokenizer)
self.assertIsInstance(tokenizer, Tokenizer)