def transform_one(self, obs, target, id): obs_tokens = nlp_utils._tokenize(obs, token_pattern) target_tokens = nlp_utils._tokenize(target, token_pattern) obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram) target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram) val_list = [] for w1 in obs_ngrams: _val_list = [] for w2 in target_ngrams: s = dist_utils._edit_dist(w1, w2) _val_list.append(s) if len(_val_list) == 0: _val_list = [ config.MISSING_VALUE_NUMERIC ] val_list.append( _val_list ) if len(val_list) == 0: val_list = [ [config.MISSING_VALUE_NUMERIC] ] return val_list
def transform_one(self, obs, target, id): return dist_utils._edit_dist(obs, target)