Exemplo n.º 1
0
 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     val_list = []
     for w1 in obs_ngrams:
         _val_list = []
         for w2 in target_ngrams:
             s = dist_utils._edit_dist(w1, w2)
             _val_list.append(s)
         if len(_val_list) == 0:
             _val_list = [ config.MISSING_VALUE_NUMERIC ]
         val_list.append( _val_list )
     if len(val_list) == 0:
         val_list = [ [config.MISSING_VALUE_NUMERIC] ]
     return val_list
Exemplo n.º 2
0
 def transform_one(self, obs, target, id):
     obs_tokens = nlp_utils._tokenize(obs, token_pattern)
     target_tokens = nlp_utils._tokenize(target, token_pattern)
     obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
     target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
     val_list = []
     for w1 in obs_ngrams:
         _val_list = []
         for w2 in target_ngrams:
             s = dist_utils._edit_dist(w1, w2)
             _val_list.append(s)
         if len(_val_list) == 0:
             _val_list = [ config.MISSING_VALUE_NUMERIC ]
         val_list.append( _val_list )
     if len(val_list) == 0:
         val_list = [ [config.MISSING_VALUE_NUMERIC] ]
     return val_list
Exemplo n.º 3
0
 def transform_one(self, obs, target, id):
     return dist_utils._edit_dist(obs, target)
Exemplo n.º 4
0
 def transform_one(self, obs, target, id):
     return dist_utils._edit_dist(obs, target)