def seg_line(line): ''' 给每一行的文本分词 ''' line = line.rstrip() #去掉每一行的换行符 words = segmentor.segment(line) #分词 postags = postagger.postag(words) #词性标注 Pos_Filter = PosFilter(words, postags) #新建一个词性过滤器对指定的词性的单词进行过滤 words_filter = Pos_Filter.filter_words() #过滤单词 rm_stop_word = StopWord.filter_words(words_filter) #去掉停用词 join_word = [ele for ele in rm_stop_word if not ele.isalpha()] join_word = ' '.join(join_word) #用空格连接分词结果 return join_word
def seg_line(line, segmentor, postagger): ''' 给每一句话的处理 ''' line = line.rstrip() #去掉每一行的换行符 words = segmentor.segment(line) #分词 postags = postagger.postag(words) #词性标注 Pos_Filter = PosFilter(words, postags) #新建一个词性过滤器对指定的词性的单词进行过滤 words_filter = Pos_Filter.filter_words() #过滤单词 rm_stop_word = StopWord.filter_words(words_filter) #去掉停用词 join_word = [ele for ele in rm_stop_word] #去掉英文单词 #join_word = [ele for ele in join_word if len(ele)>3] #去掉英文单词 return join_word