def transform_one(self, obs, target, id):
obs_tokens = nlp_utils._tokenize(obs, token_pattern)
target_tokens = nlp_utils._tokenize(target, token_pattern)
obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
val_list = []
for w1 in obs_ngrams:
_val_list = []
for w2 in target_ngrams:
s = dist_utils._edit_dist(w1, w2)
_val_list.append(s)
if len(_val_list) == 0:
_val_list = [ config.MISSING_VALUE_NUMERIC ]
val_list.append( _val_list )
if len(val_list) == 0:
val_list = [ [config.MISSING_VALUE_NUMERIC] ]
return val_list
def transform_one(self, obs, target, id):
obs_tokens = nlp_utils._tokenize(obs, token_pattern)
target_tokens = nlp_utils._tokenize(target, token_pattern)
obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram)
target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram)
val_list = []
for w1 in obs_ngrams:
_val_list = []
for w2 in target_ngrams:
s = dist_utils._edit_dist(w1, w2)
_val_list.append(s)
if len(_val_list) == 0:
_val_list = [ config.MISSING_VALUE_NUMERIC ]
val_list.append( _val_list )
if len(val_list) == 0:
val_list = [ [config.MISSING_VALUE_NUMERIC] ]
return val_list
def transform_one(self, obs, target, id):
return dist_utils._edit_dist(obs, target)
def transform_one(self, obs, target, id):
return dist_utils._edit_dist(obs, target)