Python NLTKWordTokenizer Exemples

Exemple #1

def test_nltk_word_tokenizer():
    tokenizer = NLTKWordTokenizer()

    dummy = "justo. Praesent luctus."
    assert tokenizer(dummy) == ['justo', '.', 'Praesent', 'luctus', '.']
    dummy = ""
    assert tokenizer(dummy) == []

Exemple #2

def test_ngram_tokenizer_equivalence():
    t1 = NGramsTokenizer(1)
    t2 = NLTKWordTokenizer()

    assert t1(example) == t2(example)