def test_registry_has_builtin_tokenizers(self): assert Tokenizer.by_name('word').__name__ == 'WordTokenizer' assert Tokenizer.by_name('character').__name__ == 'CharacterTokenizer'
def test_registry_has_builtin_tokenizers(self): assert Tokenizer.by_name("spacy").__name__ == "SpacyTokenizer" assert Tokenizer.by_name("character").__name__ == "CharacterTokenizer"
def test_registry_has_builtin_tokenizers(self): assert Tokenizer.by_name("word").__name__ == "WordTokenizer" assert Tokenizer.by_name("character").__name__ == "CharacterTokenizer"