Exemple #1
0
def tokenizer():
    """A tokenizer for normalizing text"""
    return Tokenizer()
Exemple #2
0
def tokenizer():
    return Tokenizer()
Exemple #3
0
def custom_tokenizer():
    return Tokenizer(app_path=APP_PATH)