Python _ngrams示例

示例#1

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     pos_list = _inter_norm_pos_list(obs_ngrams, target_ngrams)
     return pos_list

示例#2

文件： feature_intersect_position.py 项目： amsqr/Kaggle_HomeDepot

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     pos_list = _inter_norm_pos_list(obs_ngrams, target_ngrams)
     return pos_list

示例#3

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     return np_utils._try_divide(
         self._get_match_count(obs_ngrams, target_ngrams, self.idx),
         len(target_ngrams))

示例#4

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     s = 0.
     for w1 in obs_ngrams:
         for w2 in target_ngrams:
             if dist_utils._is_str_match(w1, w2, self.str_match_threshold):
                 s += 1.
     return np_utils._try_divide(s, len(obs_ngrams) * len(target_ngrams))

示例#5

文件： feature_intersect_count.py 项目： MrSnark/Kaggle_HomeDepot

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     s = 0.
     for w1 in obs_ngrams:
         for w2 in target_ngrams:
             if dist_utils._is_str_match(w1, w2, self.str_match_threshold):
                 s += 1.
     return np_utils._try_divide(s, len(obs_ngrams)*len(target_ngrams))

示例#6

文件： dist_utils.py 项目： y12uc231/BERT-1

def _get_bleu_feat(s1, s2, ngrams=3):
    if isinstance(s1, str):
        s1 = s1.split(" ")
    if isinstance(s2, str):
        s2 = s2.split(" ")
    feat = []
    for ngram in range(ngrams + 1):
        s1_ngram = ngram_utils._ngrams(s1, ngram + 1, "_")
        s2_ngram = ngram_utils._ngrams(s2, ngram + 1, "_")
        feat.append(_get_bleu(s1_ngram, s2_ngram))
    return np.array(feat, dtype=np.float32)

示例#7

文件： feature_distance.py 项目： ebernhardson/l2r

    def transform_one(self, obs, target, id):
        assert isinstance(obs, unicode)
        assert isinstance(target, unicode)

        if obs != self.last_obs:
            self.last_obs = obs
            obs_tokens = nlp_utils._tokenize(obs, token_pattern)
            self.last_obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
        target_tokens = nlp_utils._tokenize(target, token_pattern)
        target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
        return self.distance(self.last_obs_ngrams, target_ngrams)

示例#8

文件： feature_vector_space.py 项目： amsqr/Kaggle_HomeDepot

 def transform(self):
     # ngrams
     obs_ngrams = list(map(lambda x: ngram_utils._ngrams(x.split(" "), self.obs_ngram, "_"), self.obs_corpus))
     target_ngrams = list(map(lambda x: ngram_utils._ngrams(x.split(" "), self.target_ngram, "_"), self.target_corpus))
     # cooccurrence ngrams
     cooc_terms = list(map(lambda lst1,lst2: self._get_cooc_terms(lst1, lst2, "X"), obs_ngrams, target_ngrams))
     ## tfidf
     tfidf = self._init_word_ngram_tfidf(ngram=1)
     X = tfidf.fit_transform(cooc_terms)
     ## svd
     svd = TruncatedSVD(n_components=self.svd_dim, n_iter=self.svd_n_iter, random_state=config.RANDOM_SEED)
     return svd.fit_transform(X)

示例#9

文件： feature_vector_space.py 项目： qianteng/JQXXXX

 def transform(self):
     # ngrams
     obs_ngrams = list(map(lambda x: ngram_utils._ngrams(x.split(" "), self.obs_ngram, "_"), self.obs_corpus))
     target_ngrams = list(map(lambda x: ngram_utils._ngrams(x.split(" "), self.target_ngram, "_"), self.target_corpus))
     # cooccurrence ngrams
     cooc_terms = list(map(lambda lst1,lst2: self._get_cooc_terms(lst1, lst2, "X"), obs_ngrams, target_ngrams))
     ## tfidf
     tfidf = self._init_word_ngram_tfidf(ngram=1)
     X = tfidf.fit_transform(cooc_terms)
     ## svd
     svd = TruncatedSVD(n_components=self.svd_dim, 
             n_iter=self.svd_n_iter, random_state=config.RANDOM_SEED)
     return svd.fit_transform(X)

示例#10

文件： feature_stat_cooc_tfidf.py 项目： amsqr/Kaggle_HomeDepot

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     val_list = []
     for w1 in obs_ngrams:
         s = 0.
         for w2 in target_ngrams:
             if dist_utils._is_str_match(w1, w2, self.str_match_threshold):
                 s += 1.
         val_list.append(np_utils._try_divide(s, len(target_ngrams)))
     if len(val_list) == 0:
         val_list = [config.MISSING_VALUE_NUMERIC]
     return val_list

示例#11

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     val_list = []
     for w1 in obs_ngrams:
         s = 0.
         for w2 in target_ngrams:
             if dist_utils._is_str_match(w1, w2, self.str_match_threshold):
                 s += 1.
         val_list.append(np_utils._try_divide(s, len(target_ngrams)))
     if len(val_list) == 0:
         val_list = [config.MISSING_VALUE_NUMERIC]
     return val_list

示例#12

文件： feature_group_distance.py 项目： MrSnark/Kaggle_HomeDepot

 def transform_one(self, obs, target, id):
     df = self.dfTrain[self.dfTrain["search_term"] == obs].copy()
     val_list = [config.MISSING_VALUE_NUMERIC]
     if df is not None:
         df = df[df["id"] != id].copy()
         df = df[df["relevance"] == self.relevance].copy()
         if df is not None and df.shape[0] > 0:
             target_tokens = nlp_utils._tokenize(target, token_pattern)
             target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
             val_list = []
             for x in df[self.target_field]:
                 x_tokens = nlp_utils._tokenize(x, token_pattern)
                 x_ngrams = ngram_utils._ngrams(x_tokens, self.ngram)
                 val_list.append(dist_utils._jaccard_coef(x_ngrams, target_ngrams))
     return val_list

示例#13

文件： feature_stat_cooc_tfidf.py 项目： amsqr/Kaggle_HomeDepot

 def _get_avg_ngram_doc_len(self):
     lst = []
     for target in self.target_corpus:
         target_tokens = nlp_utils._tokenize(target, token_pattern)
         target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
         lst.append(len(target_ngrams))
     return np.mean(lst)

示例#14

文件： feature_group_distance.py 项目： lilkaskitc/-Forked-kaggle-HomeDepot

 def transform_one(self, obs, target, id):
     df = self.dfTrain[self.dfTrain["search_term"] == obs].copy()
     val_list = [config.MISSING_VALUE_NUMERIC]
     if df is not None:
         df = df[df["id"] != id].copy()
         df = df[df["relevance"] == self.relevance].copy()
         if df is not None and df.shape[0] > 0:
             target_tokens = nlp_utils._tokenize(target, token_pattern)
             target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
             val_list = []
             for x in df[self.target_field]:
                 x_tokens = nlp_utils._tokenize(x, token_pattern)
                 x_ngrams = ngram_utils._ngrams(x_tokens, self.ngram)
                 val_list.append(
                     dist_utils._jaccard_coef(x_ngrams, target_ngrams))
     return val_list

示例#15

 def _get_avg_ngram_doc_len(self):
     lst = []
     for target in self.target_corpus:
         target_tokens = nlp_utils._tokenize(target, token_pattern)
         target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
         lst.append(len(target_ngrams))
     return np.mean(lst)

示例#16

文件： feature_distance.py 项目： Anhmike/Kaggle_HomeDepot

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     val_list = []
     for w1 in obs_ngrams:
         _val_list = []
         for w2 in target_ngrams:
             s = dist_utils._edit_dist(w1, w2)
             _val_list.append(s)
         if len(_val_list) == 0:
             _val_list = [ config.MISSING_VALUE_NUMERIC ]
         val_list.append( _val_list )
     if len(val_list) == 0:
         val_list = [ [config.MISSING_VALUE_NUMERIC] ]
     return val_list

示例#17

文件： feature_stat_cooc_tfidf.py 项目： amsqr/Kaggle_HomeDepot

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     K = self.k1 * (1 - self.b + self.b * np_utils._try_divide(len(target_ngrams), self.avg_ngram_doc_len))
     val_list = []
     for w1 in obs_ngrams:
         s = 0.
         for w2 in target_ngrams:
             if dist_utils._is_str_match(w1, w2, self.str_match_threshold):
                 s += 1.
         bm25 = s * self._get_idf(w1) * np_utils._try_divide(1 + self.k1, s + K)
         val_list.append(bm25)
     if len(val_list) == 0:
         val_list = [config.MISSING_VALUE_NUMERIC]
     return val_list

示例#18

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     K = self.k1 * (1 - self.b + self.b * np_utils._try_divide(len(target_ngrams), self.avg_ngram_doc_len))
     val_list = []
     for w1 in obs_ngrams:
         s = 0.
         for w2 in target_ngrams:
             if dist_utils._is_str_match(w1, w2, self.str_match_threshold):
                 s += 1.
         bm25 = s * self._get_idf(w1) * np_utils._try_divide(1 + self.k1, s + K)
         val_list.append(bm25)
     if len(val_list) == 0:
         val_list = [config.MISSING_VALUE_NUMERIC]
     return val_list

示例#19

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     val_list = []
     for w1 in obs_ngrams:
         _val_list = []
         for w2 in target_ngrams:
             s = dist_utils._edit_dist(w1, w2)
             _val_list.append(s)
         if len(_val_list) == 0:
             _val_list = [ config.MISSING_VALUE_NUMERIC ]
         val_list.append( _val_list )
     if len(val_list) == 0:
         val_list = [ [config.MISSING_VALUE_NUMERIC] ]
     return val_list

示例#20

文件： feature_stat_cooc_tfidf.py 项目： amsqr/Kaggle_HomeDepot

 def _get_df_dict(self):
     # smoothing
     d = defaultdict(lambda : 1)
     for target in self.target_corpus:
         target_tokens = nlp_utils._tokenize(target, token_pattern)
         target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
         for w in set(target_ngrams):
             d[w] += 1
     return d

示例#21

 def _get_df_dict(self):
     # smoothing
     d = defaultdict(lambda : 1)
     for target in self.target_corpus:
         target_tokens = nlp_utils._tokenize(target, token_pattern)
         target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
         for w in set(target_ngrams):
             d[w] += 1
     return d

示例#22

文件： feature_first_last_ngram.py 项目： amsqr/Kaggle_HomeDepot

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     return _inter_pos_list(target_ngrams, [obs_ngrams[self.idx]])

示例#23

文件： parallel_processing.py 项目： EricTsai83/Home_Depot_Product_Search_Relevance

 def _ngram(self, text):
     tokens = text.split(" ")
     tokens = [token for token in tokens if token not in self.stopwords]
     return ngram_utils._ngrams(tokens, self.ngram, " ")

示例#24

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     return _inter_pos_list(target_ngrams, [obs_ngrams[self.idx]])

示例#25

文件： data_processor.py 项目： amsqr/Kaggle_HomeDepot

 def _ngram(self, text):
     tokens = text.split(" ")
     tokens = [token for token in tokens if token not in self.stopwords]
     return ngram_utils._ngrams(tokens, self.ngram, " ")

示例#26

文件： feature_distance.py 项目： Anhmike/Kaggle_HomeDepot

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     return dist_utils._dice_dist(obs_ngrams, target_ngrams)

示例#27

文件： feature_basic.py 项目： amsqr/Kaggle_HomeDepot

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     return np_utils._try_divide(len(set(obs_ngrams)), len(obs_ngrams))

示例#28

文件： feature_basic.py 项目： ebernhardson/l2r

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     return np_utils._try_divide(len(set(obs_ngrams)), len(obs_ngrams))

示例#29

文件： feature_first_last_ngram.py 项目： amsqr/Kaggle_HomeDepot

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     return np_utils._try_divide(self._get_match_count(obs_ngrams, target_ngrams, self.idx), len(target_ngrams))

示例#30

 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     return dist_utils._dice_dist(obs_ngrams, target_ngrams)