# coding:utf8
"""
DESC:自定义去高低词频
Author：伏草惟存
Prompt: code in Python3 env
"""
from StopWords import readFile,seg_doc
from FreqWord import *
def freqword(fdist):
wordlist =[]
print('='*3,'打印统计的词频','='*3)
for key in fdist.keys():
if fdist.get(key)>2 and fdist.get(key)<15:
wordlist.append(key+':'+str(fdist.get(key)))
return wordlist
if __name__=='__main__':
# 1 读取文本
path= r'../Corpus/CSCMNews/体育/0.txt'
str_doc = readFile(path)
word_list =seg_doc(str_doc)
# 2 选择高低词
fdist = nltk_wf_feature(word_list)
wordlist=freqword(fdist)
print(wordlist)
# for key, value in dictonary.items():
# print(corpus_tfidf)
return corpus_tfidf
# 将每一类保存到文件中
# catgs = list(corpus_tfidf.keys()) # ['体育', '时政']
# for catg in catgs:
# savepath = r'./'
# corpora.MmCorpus.serialize(r'{f}{s}{c}.mm'.format(f=savepath, s=os.sep, c=catg), corpus_tfidf.get(catg), id2word=dictonary)
if __name__ == '__main__':
path = r"../CSCMNews/体育/0.txt"
str_doc = readFile(path)
word_list = seg_doc(str_doc)
word_str = " ".join(word_list)
path2 = r"../CSCMNews/时政/339764.txt"
str_doc2 = readFile(path2)
word_list2 = seg_doc(str_doc2)
# print(type(word_list2))
word_str2 = " ".join(word_list2)
corpus = [] # 每个文本为字符串
classVec = ["体育", "时政"]
corpus.append(word_str)
corpus.append(word_str2)
# 使用sklearn 计算词的tfidf
a = tfidf_sklearn_feature(
corpus, classVec) # len(a["体育"][0]), len(a["时政"][0]) 242, 242
# 该类会将文本中的词语转换为词频矩阵，矩阵元素a[i][j] 表示j词在i类文本下的词频
vectorizer = CountVectorizer() # 构建词汇表
transformer = TfidfTransformer() # 该类会统计每个词语的tf-idf权值
tfidf = transformer.fit_transform(vectorizer.fit_transform(corpus)) # 第一个fit_transform是计算tf-idf，第二个fit_transform是将文本转为词频矩阵
word = vectorizer.get_feature_names() # 获取词袋模型中的所有词语
weight = tfidf.toarray() # 将tf-idf矩阵抽取出来，元素a[i][j]表示j词在i类文本中的tf-idf权重
for i in range(len(weight)): # 打印每类文本的tf-idf词语权重，第一个for遍历所有文本，第二个for便利某一类文本下的词语权重
print(u"-------这里输出第", i, u"类文本的词语tf-idf权重------")
for j in range(len(word)):
print(word[j], weight[i][j])
if __name__=='__main__':
corpus =[]
# corpus参数样例数据如下：
# corpus = ["我 来到 成都 成都 春熙路 很 开心",
# "今天 在 宽窄巷子 耍 了 一天 ",
# "成都 整体 来说 还是 挺 安逸 的",
# "成都 的 美食 真 巴适 惨 了"]
# 1 读取文本
path1= r'../dataSet/CSCMNews/体育/0.txt'
str_doc1 = readFile(path1)
word_list1 =' '.join(seg_doc(str_doc1))
path2= r'../dataSet/CSCMNews/时政/339764.txt'
str_doc2 = readFile(path2)
word_list2 =' '.join(seg_doc(str_doc2))
corpus.append(word_list1)
corpus.append(word_list2)
sklearn_tfidf_feature(corpus)
catg = folder.split(os.sep)[-1]
for file in os.listdir(folder): # level2 directory
file_path = os.path.join(folder, file)
# 文件具体操作
if os.path.isfile(file_path):
this_file = open(file_path, 'rb') # rb读取更快
content = this_file.read().decode('utf8')
yield catg, content
this_file.close()
if __name__ == '__main__':
start = time.time()
filepath = os.path.abspath(
r'C:\Users\Chengyu.Dean\Desktop\树维\江西省挖煤\new_江西省报告')
files = loadFiles(filepath)
# n = 5 # n表示抽样率
for i, msg in enumerate(files):
catg = msg[0]
content = msg[1]
content = seg_doc(content)
print(
'{t}***{i}\t docs has been dealed'.format(i=i,
t=time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime())),
'\n', catg, ':\t', content[:20])
end = time.time()
print('total cost time: %.2f' % (end - start) + 's')