def _get_match_count(self, obs, target, idx): cnt = 0 if (len(obs) != 0) and (len(target) != 0): for word in target: if dist_utils._is_str_match(word, obs[idx], self.str_match_threshold): cnt += 1 return cnt
def transform_one(self, obs, target, id): obs_tokens = nlp_utils._tokenize(obs, token_pattern) target_tokens = nlp_utils._tokenize(target, token_pattern) obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram) target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram) s = 0. for w1 in obs_ngrams: for w2 in target_ngrams: if dist_utils._is_str_match(w1, w2, self.str_match_threshold): s += 1. return np_utils._try_divide(s, len(obs_ngrams)*len(target_ngrams))
def transform_one(self, obs, target, id): obs_tokens = nlp_utils._tokenize(obs, token_pattern) target_tokens = nlp_utils._tokenize(target, token_pattern) obs_nterms = ngram_utils._nterms(obs_tokens, self.nterm) target_nterms = ngram_utils._nterms(target_tokens, self.nterm) s = 0. for w1 in obs_nterms: for w2 in target_nterms: if dist_utils._is_str_match(w1, w2, self.str_match_threshold): s += 1. return np_utils._try_divide(s, len(obs_nterms) * len(target_nterms))
def transform_one(self, obs, target, id): obs_tokens = nlp_utils._tokenize(obs, token_pattern) target_tokens = nlp_utils._tokenize(target, token_pattern) obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram) target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram) s = 0. for w1 in obs_ngrams: for w2 in target_ngrams: if dist_utils._is_str_match(w1, w2, self.str_match_threshold): s += 1. break return s
def transform_one(self, obs, target, id): obs_tokens = nlp_utils._tokenize(obs, token_pattern) target_tokens = nlp_utils._tokenize(target, token_pattern) obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram) target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram) val_list = [] for w1 in obs_ngrams: s = 0. for w2 in target_ngrams: if dist_utils._is_str_match(w1, w2, self.str_match_threshold): s += 1. val_list.append(np_utils._try_divide(s, len(target_ngrams))) if len(val_list) == 0: val_list = [config.MISSING_VALUE_NUMERIC] return val_list
def transform_one(self, obs, target, id): obs_tokens = nlp_utils._tokenize(obs, token_pattern) target_tokens = nlp_utils._tokenize(target, token_pattern) obs_ngrams = ngram_utils._ngrams(obs_tokens, self.ngram) target_ngrams = ngram_utils._ngrams(target_tokens, self.ngram) K = self.k1 * (1 - self.b + self.b * np_utils._try_divide(len(target_ngrams), self.avg_ngram_doc_len)) val_list = [] for w1 in obs_ngrams: s = 0. for w2 in target_ngrams: if dist_utils._is_str_match(w1, w2, self.str_match_threshold): s += 1. bm25 = s * self._get_idf(w1) * np_utils._try_divide(1 + self.k1, s + K) val_list.append(bm25) if len(val_list) == 0: val_list = [config.MISSING_VALUE_NUMERIC] return val_list