def main(): input_s = t2s.convert(args.input) words = pseg.cut(input_s) word_list = [] for word, flag in words: if word is not ' ': word_dict = {'word': s2t.convert(word), 'tag': flag} word_list.append(word_dict) result = False for word in word_list: for keyword in keywords: if word['tag'] in 'eng': distance = jaro_winkler_similarity(word['word'], keyword) if distance > threshold: result = True else: if word['word'] in keyword: result = True if result is True: print("Match") else: print("Not match")
def hide_word(text, words): """ Заменяет слова words знаками вопроса для теста """ start_word = words eng = 'abcdefghijklmnopqrstuvwxyz' words_to_replace = [] text = text.replace('́', '') words = re.sub(r'\(.*?\)', '', words).strip() words = [words] + re.split(r'[\s\.\,«»]+', words) norm_words = [morph.parse(word) for word in words] for word in word_tokenize(text): for a in norm_words: for a1 in a: for b in morph.parse(word): # print( # word, a1.normal_form, b.normal_form, # distance.edit_distance(a1.normal_form, b.normal_form), # distance.jaro_similarity(a1.normal_form, b.normal_form), # distance.jaro_winkler_similarity(a1.normal_form, b.normal_form), # ) #print(word, a1.normal_form, b.normal_form, ss.stem(word), a1.normal_form == b.normal_form) if a1.normal_form == b.normal_form: words_to_replace.append(word) elif ss.stem(a1.normal_form) == ss.stem(b.normal_form): words_to_replace.append(word) elif distance.jaro_winkler_similarity( a1.normal_form, b.normal_form) >= 0.80: words_to_replace.append(word) words_to_replace = list(set(words_to_replace + [start_word])) # print(words_to_replace) # print(sorted(words_to_replace, key=len, reverse=True)) for w in sorted(words_to_replace, key=len, reverse=True): text = text.replace(w, '???') text = re.sub(r'[\?\»][\?\s]+[\?\»]', '???', text) return text
def get_best_match(ent, p_list): max_val = -1 max_entity = "" max_list = list() for p in p_list: sim_val = jaro_winkler_similarity(ent, p[0]) log("\t\tSim value between : " + ent + " | " + p[0] + " => " + str(sim_val)) if max_val < sim_val: max_val = sim_val max_entity = p for p in p_list: sim_val = jaro_winkler_similarity(ent, p[0]) if sim_val == max_val: max_list.append([max_val, p]) return max_list
def isKeyword(tokens, template): """Check if there is a keyword match in tokens""" result = False for token in tokens: if token["tag"] in "eng": for keyword in template["eng"]: distance = jaro_winkler_similarity(token["word"], keyword) if distance >= template["threshold"]: result = True else: for keyword in template["cht"]: if token["word"] in keyword: result = True return result
def get_dists(keyword): dists = [] for word in words_preprocessed: dists.append({ "edit_dist": edit_distance(word, keyword), "jaro_simi": jaro_similarity(word, keyword), "jaro_winkler_simi": jaro_winkler_similarity(word, keyword), "jaccard_dist": jaccard_distance(set(word), set(keyword)), "word": word, "keyword": keyword }) return pd.DataFrame(dists).sort_values("edit_dist").iloc[0:3, :]
def run_jaro_winkler_similarity(lhs, entities): max_similarity = [ ('', '', 0.0), ] for entity in entities: similarity = distance.jaro_winkler_similarity(lhs, entity['author']) if max_similarity[0][2] < similarity: max_similarity = [ (entity['author'], entity['url'], similarity), ] elif max_similarity[0][2] == similarity: max_similarity.append( (entity['author'], entity['url'], similarity)) print( 'Jaro Winkler similarity with {0} and {1} entities results in {2} miximum similarity of {3}' .format(lhs, len(entities), len(max_similarity), max_similarity[0][2])) return max_similarity
def is_rough_match(text, name, thresh = 0.8): text = text.lower() name = name.lower() for sw in STOP_WORDS: text = re.sub(' +', ' ', re.sub('(^| )' + sw + '($| )', ' ', text)).strip() name = re.sub(' +', ' ', re.sub('(^| )' + sw + '($| )', ' ', name)).strip() if text == name: return Match.Full splits = name.split(' ') for i in range(len(splits)): if text == ' '.join(splits[i:]): return Match.Partial if jaro_winkler_similarity(text, ' '.join(splits[i:])) > thresh: return Match.Partial return Match.NoMatch
def nltk_jarowinklersimilarity(parent: str, child: str): return np.asarray([[jaro_winkler_similarity(parent, child)]])
def nltk_jarowinklersimilarity(parent: str, child: str): return jaro_winkler_similarity(parent, child)