def test_replace_keywords(self): """For each of the test case initialize a new KeywordProcessor. Add the keywords the test case to KeywordProcessor. Replace keywords and check if they match the expected result for the test case. """ for test_id, test_case in enumerate(self.test_cases): keyword_replacer = KeywordProcessor() keyword_replacer.add_keywords_from_dict(test_case['keyword_dict']) new_sentence = keyword_replacer.replace_keywords( test_case['sentence']) replaced_sentence = test_case['sentence'] keyword_mapping = {} for val in test_case['keyword_dict']: for value in test_case['keyword_dict'][val]: keyword_mapping[value] = val for key in sorted(keyword_mapping, key=len, reverse=True): lowercase = re.compile(r'(?<!\w){}(?!\w)'.format( re.escape(key))) replaced_sentence = lowercase.sub(keyword_mapping[key], replaced_sentence) self.assertEqual( new_sentence, replaced_sentence, "new_sentence don't match the expected results for test case: {}" .format(test_id))
def test_file_format_two(self): keyword_processor = KeywordProcessor() keyword_processor.add_keyword_from_file('test/keywords_format_two.txt') sentence = 'I know java and product management' keywords_extracted = keyword_processor.extract_keywords(sentence) self.assertEqual(keywords_extracted, ['java', 'product management'], "Failed file format one test") sentence_new = keyword_processor.replace_keywords(sentence) self.assertEqual(sentence_new, "I know java and product management", "Failed file format one test")
def test_list_loading(self): keyword_processor = KeywordProcessor() keyword_list = ["java", "product management"] keyword_processor.add_keywords_from_list(keyword_list) sentence = 'I know java and product management' keywords_extracted = keyword_processor.extract_keywords(sentence) self.assertEqual(keywords_extracted, ['java', 'product management'], "Failed file format one test") sentence_new = keyword_processor.replace_keywords(sentence) self.assertEqual(sentence_new, "I know java and product management", "Failed file format one test")
def fastcleaner(docs, replacewords): ''' 语料清洗工具, FastText可用于快速进行大规模语料库的文本搜索与替换 INPUT -> 文档集(词之间为空格)、替换词表 ''' docs_new = [] keyword_processor = KeywordProcessor() for doc in docs: for word1, word2 in replacewords: keyword_processor.add_keyword(word1, word2) # 前面一个词为定位词, 后面一个词为替换 docs_new.append(keyword_processor.replace_keywords(doc)) return docs_new
def test_dictionary_loading(self): keyword_processor = KeywordProcessor() keyword_dict = { "java": ["java_2e", "java programing"], "product management": ["product management techniques", "product management"] } keyword_processor.add_keywords_from_dict(keyword_dict) sentence = 'I know java_2e and product management techniques' keywords_extracted = keyword_processor.extract_keywords(sentence) self.assertEqual(keywords_extracted, ['java', 'product management'], "Failed file format one test") sentence_new = keyword_processor.replace_keywords(sentence) self.assertEqual(sentence_new, "I know java and product management", "Failed file format one test")
# coding=utf-8 # if len(keywordList) > 500: exec(flashtext) else: exec(Regex) from flashtext.keyword import KeywordProcessor keyword_processor = KeywordProcessor() keyword_processor.add_keyword('PyTorch') keyword_processor.add_keyword(keyword='and', clean_name='or') keywords_found = keyword_processor.extract_keywords('I love Python and PyTorch.') print(keywords_found) # ['or', 'PyTorch'] keyword_processor.add_keyword(keyword='Python', clean_name='Tensorflow') new_sentence = keyword_processor.replace_keywords('I love Python and PyTorch.') print(new_sentence) # I love Tensorflow or PyTorch.
unique_keywords_sublist = list( set(random.sample(all_words, keywords_length))) # compile regex # source: https://stackoverflow.com/questions/6116978/python-replace-multiple-strings rep = dict([(key, '_keyword_') for key in unique_keywords_sublist]) compiled_re = re.compile("|".join(rep.keys())) # add keywords to flashtext keyword_processor = KeywordProcessor() for keyword in unique_keywords_sublist: keyword_processor.add_keyword(keyword, '_keyword_') # time the modules start = time.time() _ = keyword_processor.replace_keywords(story) mid = time.time() _ = compiled_re.sub(lambda m: rep[re.escape(m.group(0))], story) end = time.time() # print output print( str(keywords_length).ljust(6), '|', "{0:.5f}".format(mid - start).ljust(9), '|', "{0:.5f}".format(end - mid).ljust(9), '|', ) # Count | FlashText | Regex # -------------------------------
from flashtext.keyword import KeywordProcessor # 提取关键字 # add_keyword(查找字符,替换字符),也就是先找到句子中的’你好’,然后显示出来的是add_keyword的替换字符 # 英文 keyword_processor = KeywordProcessor() keyword_processor.add_keyword('Big Apple', 'New York') keyword_processor.add_keyword('Bay Area') keywords_found = keyword_processor.extract_keywords( 'I love Big Apple and Bay Area.') print(keywords_found) #中文 keyword_processor = KeywordProcessor() keyword_processor.add_keyword('你好', '您好') # 前面一个单词为住 后面一个单词为替换 keyword_processor.add_keyword('不要') keywords_found = keyword_processor.extract_keywords( '你好,请不要随便践踏草坪。') #显示的单词为替换之后的 print(keywords_found) # 替换关键字 keyword_processor = KeywordProcessor() keyword_processor.add_keyword('你好', '您好') # 前面一个单词为住 后面一个单词为替换 new_sentence = keyword_processor.replace_keywords('你好,请不要随便践踏草坪。') print(new_sentence)