def test_word_based_ngrams(): text = 'Hello this is a sentence' assert [('Hello',), ('this',), ('is',), ('a',), ('sentence',)] == \ utils.word_based_ngrams(text, 1) assert [('Hello', 'this'), ('this', 'is'), ('is', 'a'), ('a', 'sentence')] == \ utils.word_based_ngrams(text, 2)
def word_based_4gram(text): return utils.word_based_ngrams(text, n, padding_left=padding, padding_right=padding)
def word_based_4gram(text): return utils.word_based_ngrams( text, n, padding_left=padding, padding_right=padding)