def get_all_ngrams(tokens):
if not tokens:
return []
key = "<||>".join(tokens)
if key not in _NGRAMS_CACHE:
ngrams = compute_all_ngrams(tokens, len(tokens))
_NGRAMS_CACHE[key] = ngrams
return deepcopy(_NGRAMS_CACHE[key])
def get_all_ngrams(tokens):
from snips_nlu_utils import compute_all_ngrams
if not tokens:
return []
key = "<||>".join(tokens)
if key not in _NGRAMS_CACHE:
ngrams = compute_all_ngrams(tokens, len(tokens))
_NGRAMS_CACHE[key] = ngrams
return deepcopy(_NGRAMS_CACHE[key])
def test_should_compute_all_ngrams(self):
# Given
tokens = ["hello", "beautiful", "world", "!"]
# When
ngrams = compute_all_ngrams(tokens, 3)
# Then
expected_ngrams = [{
'ngram': 'hello',
'token_indexes': [0]
}, {
'ngram': 'hello beautiful',
'token_indexes': [0, 1]
}, {
'ngram': 'hello beautiful world',
'token_indexes': [0, 1, 2]
}, {
'ngram': 'beautiful',
'token_indexes': [1]
}, {
'ngram': 'beautiful world',
'token_indexes': [1, 2]
}, {
'ngram': 'beautiful world !',
'token_indexes': [1, 2, 3]
}, {
'ngram': 'world',
'token_indexes': [2]
}, {
'ngram': 'world !',
'token_indexes': [2, 3]
}, {
'ngram': '!',
'token_indexes': [3]
}]
self.assertListEqual(expected_ngrams, ngrams)
def get_all_ngrams(tokens):
key = "<||>".join(tokens)
if key not in _NGRAMS_CACHE:
ngrams = compute_all_ngrams(tokens, len(tokens))
_NGRAMS_CACHE[key] = ngrams
return _NGRAMS_CACHE[key]
def get_all_ngrams(tokens):
key = "<||>".join(tokens)
if key not in _NGRAMS_CACHE:
ngrams = compute_all_ngrams(tokens, len(tokens))
_NGRAMS_CACHE[key] = ngrams
return _NGRAMS_CACHE[key]