def wordgrams(sent, depth, pc, th, ct, et, ip, d=0): if depth == 0: return sent, None else: """Builds word grams according to the specification.""" phrases = Phrases(sent, common_terms=ct, min_count=pc, threshold=th) grams = Phraser(phrases) grams.phrasegrams = exclude_words(grams.phrasegrams, et) d += 1 if d < depth: return wordgrams(grams[sent], depth, pc, th, ct, et, ip, d) else: return grams[sent], grams
def _wordgrams(self, sent, depth, pc, th, d=0): """ Builds word grams according to the specification. (Written by mat2vec) """ if depth == 0: return sent, None else: phrases = Phrases(sent, common_terms=COMMON_TERMS, min_count=pc, threshold=th) grams = Phraser(phrases) grams.phrasegrams = self._exclude_words(grams.phrasegrams, EXCLUDE_PUNCT) d += 1 if d < depth: return self._wordgrams(grams[sent], depth, pc, th, d) else: return grams[sent], grams