def find_sentences(keywords_list, input_str, topn): from flashtext.keyword import KeywordProcessor from collections import Counter ''' :param keywords_list: list ['word1',..'wordn'] :param doc_txt: txt文档 :param topn: 显示重要程度TOPN的句子 :return:keywords_sentences: dict {'word1':['sen1','sen2',...],...} ''' keyword_processor = KeywordProcessor() keyword_processor.add_keywords_from_list(keywords_list) # 按标点符号切分句子 doc_cut_list = set(re.split('[?!…。?!]', input_str)) # 去重 # 查找关键词所在句子 sentences_wordcount = {} for sentence in doc_cut_list: keywords_found = keyword_processor.extract_keywords(sentence) if len(keywords_found) != 0: keywords_count = Counter(keywords_found) sentences_wordcount[sentence] = keywords_count keywords_sentences = {} for word in keywords_list: keywords_sentences[word] = [ k for k, v in sentences_wordcount.items() if word in v.keys() ][0:topn] return keywords_sentences
def test_list_loading(self): keyword_processor = KeywordProcessor() keyword_list = ["java", "product management"] keyword_processor.add_keywords_from_list(keyword_list) sentence = 'I know java and product management' keywords_extracted = keyword_processor.extract_keywords(sentence) self.assertEqual(keywords_extracted, ['java', 'product management'], "Failed file format one test") sentence_new = keyword_processor.replace_keywords(sentence) self.assertEqual(sentence_new, "I know java and product management", "Failed file format one test")
def create_keywordProcessor(list_of_terms, remove_stopwords=True, custom_stopword_list=[""]): """ Creates a new flashtext KeywordProcessor and optionally does some lightweight text cleaning to remove stopwords, including any provided by the user. """ # create a KeywordProcessor keyword_processor = KeywordProcessor() keyword_processor.add_keywords_from_list(list_of_terms) # remove English stopwords if requested if remove_stopwords == True: keyword_processor.remove_keywords_from_list(stopwords.words('english')) # remove custom stopwords keyword_processor.remove_keywords_from_list(custom_stopword_list) return (keyword_processor)
def find_sentences_weight(word_weight_dict, input_str, topn): from flashtext.keyword import KeywordProcessor from collections import Counter ''' :param keywords_list: list ['word1',..'wordn'] :param top_keywords: 计算TOP权重的关键词 :param input_str: str文档 :param topn: 显示重要程度TOPN的句子 :return:keywords_sentences: dict {'word1':['sen1','sen2',...],...} ''' # 读取词汇权重文件 keywords_list = list(word_weight_dict.keys()) keyword_processor = KeywordProcessor() keyword_processor.add_keywords_from_list(list(keywords_list)) input_str = re.sub('\s', '', input_str) doc_cut_list = set(re.split('[?!…。;?!]', input_str)) # 去重 # 计算句子重要程度 sentences_wordcount = {} # [('句子',(value,['word1',...'wordn']))] for sentence in doc_cut_list: keywords_found = keyword_processor.extract_keywords(sentence) if len(keywords_found) != 0: keywords_in_sentence = [] sentence_value = 0 keywords_count = Counter(keywords_found) for k, v in keywords_count.items(): sentence_value = sentence_value + v * word_weight_dict[ k] # 权重 = 求和(句子中出现的关键词权重) keywords_in_sentence.append(k) sentence_value = len(keywords_in_sentence) * sentence_value / len( sentence) # 权重 = 关键词个数 * 关键词权重 / 句子长度, 可根据需要注释 sentences_wordcount[sentence] = (sentence_value, keywords_in_sentence) sentences_wordcount_sort = sorted( sentences_wordcount.items(), key=lambda x: x[1][0], reverse=True) # 句子,权重 [('句子',(value,['word1',...'wordn']))] topn_sentences = [s[0] for s in sentences_wordcount_sort[0:topn]] return topn_sentences
def find_sentences_rule(keywords_list, doc_txt): from flashtext.keyword import KeywordProcessor keyword_processor = KeywordProcessor() keyword_processor.add_keywords_from_list(keywords_list) # 按标点符号切分句子 fie = [ line.strip() for line in open(doc_txt, 'r', encoding='utf-8').readlines() if len(line) > 1 ] # 按特殊标点符号,分割句子 article = '' for i in fie: article += i.strip() + '*' doc_cut_list = re.split('[?!…。?!*]', article) # 对不同行,没有标点的句子通过‘* sentences_important = [] for sentence in doc_cut_list: keywords_found = keyword_processor.extract_keywords(sentence) if len(keywords_found) != 0: sentences_important.append(sentence) print(sentences_important) return sentences_important
from flashtext.keyword import KeywordProcessor keyword_processor = KeywordProcessor() # keyword_processor.add_keyword('SQL injection') keyword_processor.add_keyword('SQL injection', ('vulnerability type', 'SQL injection')) keyword_processor.add_keyword('cross-site scripting', ('vulnerability type', 'cross-site scripting')) keyword_processor.add_keyword('cross-site scripting', 'XSS') # keyword_processor.add_keyword('parameter') # keyword_processor.add_keyword('function') # keyword_processor.add_keyword('variable') keyword_dict = {"cross-site scripting": ["XSS"], "parametert": ["variabler"]} # {'clean_name': ['list of unclean names']} keyword_processor.add_keywords_from_dict(keyword_dict) # Or add keywords from a list: keyword_processor.add_keywords_from_list(["parameter", "function", "variable"]) ####keyword replacement # keyword_processor.add_keyword('cross-site scripting', 'XSS') # keyword_processor.replace_keywords('vulnerability is cross-site scripting') keyword_processor.extract_keywords( 'SQL injection vulnerability in the update_zone function in catalog/admin/geo_zones.php in osCommerce Online Merchant 2.3.3.4 and earlier allows remote administrators to execute arbitrary SQL commands via the zID parameter in a list action. ' )
import requests from flashtext.keyword import KeywordProcessor from nltk.corpus import stopwords # let's read in a couple of forum posts forum_posts = pd.read_csv("../input/ForumMessages.csv") # get a smaller sub-set for playing around with sample_posts = forum_posts.Message[0:3] # get data from list of top 5000 pypi packages (last 30 days) url = 'https://hugovk.github.io/top-pypi-packages/top-pypi-packages-30-days.json' data = requests.get(url).json() # get just the list of package names list_of_packages = [data_item['project'] for data_item in data['rows']] # create a KeywordProcess keyword_processor = KeywordProcessor() keyword_processor.add_keywords_from_list(list_of_packages) # remove english stopwords keyword_processor.remove_keywords_from_list(stopwords.words('english')) # remove custom stopwords keyword_processor.remove_keywords_from_list(['http','kaggle']) # test our keyword processor for post in sample_posts: keywords_found = keyword_processor.extract_keywords(post, span_info=True) print(keywords_found)
class SimWordCount(object): def __init__(self, input, keywords_list=None): # 初始化词频统计类 self.keyword_processor = KeywordProcessor() # 加载停用词词典 stopwords = os.sep.join( [os.path.dirname(__file__), 'lib', 'stopwords_all.txt']) if not os.path.exists(stopwords): raise FileNotFoundError('stopwords file is not found!') self.stopwords_list = [ line.strip().decode('utf-8') for line in open(stopwords, 'rb').readlines() ] # 加入自定义词,辅助分词 if keywords_list: for word in keywords_list: jieba.add_word(word) # 加载文档并分词 self.seg_text = self._doc_cut(input) # 分词 # 加载近义词词典 cilin = os.sep.join( [os.path.dirname(__file__), 'lib', 'new_cilin.txt']) if not os.path.exists(cilin): raise FileNotFoundError('近义词词典new_cilin.txt is not found!') self.file = cilin self.word_code = {} self.code_word = {} self.vocab = set() self._read_cilin() # 加载预定义的近似词词典 if os.path.exists( os.sep.join([os.path.dirname(__file__), 'lib', 'simword.txt'])): self.simword_group = [ line.strip().decode('utf-8-sig') for line in open( os.sep.join([ os.path.dirname(__file__), 'lib', 'simword.txt' ]), 'rb').readlines() ] self.simword_group = [ group.split() for group in self.simword_group ] else: raise FileNotFoundError('可在当前文件夹下增加过滤词simword.txt,近义词中的词群将被列为统计结果') # 加载过滤词表 if os.path.exists( os.sep.join( [os.path.dirname(__file__), 'lib', 'black_list.txt'])): self.blacklist = [ line.strip().decode('utf-8') for line in open( os.sep.join([ os.path.dirname(__file__), 'lib', 'black_list.txt' ]), 'rb').readlines() ] else: raise FileNotFoundError( '可在当前文件夹下增加黑名单black_list.txt,黑名单中的词将不被列为统计结果') # 加载分词词典 dict_path = os.sep.join( [os.path.dirname(__file__), 'lib', 'new_dict.txt']) try: jieba.load_userdict(dict_path) except FileNotFoundError: print('WARNING: 为了更好的分词效果,建议在当前目录放置new_dict.txt自定义词典,辅助jieba分词') finally: pass def count(self, input, use_filter_words=True, topn=10, output_limit=1, count_only_singleword=True): ''' :param : input:输入文本 type: str :param : thres:近义词阈值 0-1 type: float 词编码相同比例阈值, thres%相同,则认为是近义词 :param : use_filter_words:是否使用过滤词表 True / False 默认为True :return: textgroup_count_sort 返回近义词组统计信息 type: list [(words_group,count),...] ''' # 输入参数类型检查 if topn < 1: raise ValueError('topn 必须大于等于1') if output_limit < 1: raise ValueError('topn 必须大于等于1') if use_filter_words == 'True': use_filter_words = True elif use_filter_words == 'False': use_filter_words = False else: raise ValueError('use_filter_words必须是True或False') words = set(self.seg_text) # 原文中的全部词 # 计算每个词的位置因子 if count_only_singleword: # 只统计单个词频 words_from_dict = set() else: # 统计单个词,包括近义词群 words_from_dict = set(self.word_code.keys()) # 近义词词典中包含的词(非重复) words_in_simgroup = set([ word for elem in self.simword_group for word in elem ]) # 自定义词表中包含的词(非重复) # words_notin_group = [[word] for word in list(words - words_from_dict-words_in_simgroup) if len(word) >output_limit and word[0]!='.'] # 不在1.近义词词典 2.自定义词群中的词(非重复) words_notin_group = [ [word] for word in list(words - words_from_dict - words_in_simgroup) if len(word) > output_limit and word >= '\u4e00' and word <= '\u9fff' ] # 不在1.近义词词典 2.自定义词群中的词(非重复) 去除数字,字符,单字 # 将自定义词加入词典 dict_simgroup_result = {} if self.simword_group: self.keyword_processor.add_keywords_from_list( list(words_in_simgroup)) keywords_found = self.keyword_processor.extract_keywords(input) keywords_count = Counter(keywords_found) # 统计自定义词群频率 for l in self.simword_group: word_count = 0 for w in l: word_count += keywords_count[w] if word_count != 0: s = ','.join(l) dict_simgroup_result[s] = word_count # 采用黑名单机制对返回结果词频进行过滤 if use_filter_words: words_notin_filter = [word[0] for word in words_notin_group] words_notin_group = list( set(words_notin_filter) - set(self.blacklist)) words_notin_group = [[word] for word in words_notin_group] words_from_dict = set(self.word_code.keys()) - set(self.blacklist) self.simword_group = [ list(set(group) - set(self.blacklist)) for group in self.simword_group ] # 如果自定义词表 text_word_dict = {} for word in words: if word in words_from_dict: if len(word) > output_limit: text_word_dict[word] = self.word_code[word] words_group = self._get_words_group(text_word_dict) # 近义词词群 words_group = words_notin_group + words_group # 需要统计词频的全部词(普通词,近义词词群),自定义近义词由flashtext自己统计 dict_result = {} count = Counter(self.seg_text) for l in words_group: word_count = 0 for w in l: word_count += count[w] s = ','.join(l) dict_result[s] = word_count if self.simword_group: all_dict_result = dict(dict_result, **dict_simgroup_result) else: all_dict_result = dict_result textgroup_count_sort = sorted(all_dict_result.items(), key=lambda x: x[1], reverse=True) # 按照出现频率排序的词群 # keys = [group[0] for group in textgroup_count_sort[0:topn]] keys = [group[0] for group in textgroup_count_sort] values = [group[1] for group in textgroup_count_sort] # json_result = [{'key': group[0], 'value': group[1]} for group in textgroup_count_sort[0:topn]] return keys, values def count_span(self): # 计算每个词在文章中的跨度 doc_cut = self.seg_text words = set(doc_cut) # 原文中的全部词 span_word = {} sum_span = len(doc_cut) # 文章中词的总个数 for word in words: first = doc_cut.index(word) # 在文章中首次出现的位置 doc_cut.reverse() last = sum_span - doc_cut.index(word) # 在文章中最后出现的位置 doc_cut.reverse() span_word[word] = (last - first + 1) / sum_span return span_word def _read_cilin(self): with open(self.file, 'r', encoding='gbk') as f: for line in f.readlines(): res = line.split() code = res[0] # 词义编码 words = res[1:] # 同组的多个词 self.vocab.update(words) # 一组词更新到词汇表中 self.code_word[code] = words # 字典,目前键是词义编码,值是一组单词。 for w in words: if w in self.word_code.keys(): # 最终目的:键是单词本身,值是词义编码。 self.word_code[w].append(code) # 如果单词已经在,就把当前编码增加到字典中 else: self.word_code[w] = [code] # 反之,则在字典中添加该项。 @staticmethod def _get_words_group(word_label_dict): words_group = [] values = [t for t in set(tuple(_) for _ in word_label_dict.values())] # 去重 for value in values: words = [ k for k, v in word_label_dict.items() if tuple(v) == value ] # 编码完全一样 words_group.append(words) return words_group @staticmethod def _read_words_from_file(fn): f = open(fn, encoding='utf-8').readlines() words = [s.strip() for s in f if s.strip()] return words def _doc_cut(self, doc): seg_text = [] for word in jieba.cut(doc, cut_all=False): if not word.strip(): continue if word.strip() in self.stopwords_list: continue seg_text.append(word) return seg_text
langs = ["en", "fr", "nl"] domains = ["corp", "equi", "htfl", "wind"] for lang in langs: mt = MosesTokenizer(lang=f'{lang}') for domain in domains: sourcedir = f"{workdrive}/coding/Text2TCS/git/ACTER/{lang}/{domain}/annotations/" list_of_terms = [] with open(f"{sourcedir}/{domain}_{lang}_terms_nes.ann", "r", encoding="utf-8") as f: for line in f.readlines(): s = str(line).replace("OOD_Term", "").replace("Common_Term", "").replace("Specific_Term", "").replace( "Named_Entity", "").strip("\n").strip("\t") list_of_terms.append(s) kwp.add_keywords_from_list(list_of_terms) # Remove unwanted terms from list (single letters, prepositions, stop-words etc.) abc_list = list(string.ascii_uppercase + string.ascii_lowercase) kwp.remove_keywords_from_list(abc_list) kwp.remove_keywords_from_list(word_boundary_list) for i in stopwords(f"{lang}"): kwp.remove_keyword(i) kwp.remove_keyword(i.capitalize()) # Extract the terms with open(f"{outdir}/{lang}{suffix1}/{domain}{suffix2}full_tok.txt", "r", encoding="utf-8") as f: sentences = f.readlines() # print(sentences[-10:]) results =[]