Beispiel #1
0
 def f(text):
   return [' '.join(sorted(ngram)).lower() for ngram in ngrams(n, tokenize(text))]
Beispiel #2
0
 def f(text):
   return [ngram for w in tokenize(text) for ngram in ngrams(n, '$' + w.lower() + '$')]
Beispiel #3
0
 def f(row):
   return [' '.join(ngram) for ngram in ngrams(n, [tok[2].lower() + '_' + tok[1] for tok in getToks(row) if 1 < len(tok[2])])]
Beispiel #4
0
 def f(row):
   return [' '.join(ngram) for ngram in ngrams(n, [tok[1] for tok in getToks(row)])]
Beispiel #5
0
 def f(row):
   return [' '.join(sorted(ngram)) for ngram in ngrams(n, [tok[2].lower() for tok in getToks(row)])]
Beispiel #6
0
def mixedNgramsTTS(row):
  return ['%s %s %s' % (ngram[0][1], ngram[1][1], ngram[2][2]) for ngram in ngrams(3, list(getToks(row)))]
Beispiel #7
0
def mixedNgramsST(row):
  return ['%s %s' % (ngram[0][2], ngram[1][1]) for ngram in ngrams(2, list(getToks(row)))]