def build_wordcloud(msg): msg = ','.join([m.split(DELIMITER)[-1]for m in msg]) words = list(jieba.cut(msg)) for i in range(len(words)): if words[i] in stopwords: words[i] = '' wc = WC(font_path='./resource/msyh.ttf', #如果是中文必须要添加这个,否则乱码 background_color='white', width=1000, height=800,).generate(' '.join(words)) wc.to_file('./output/chat/message_cloud.png') return './output/chat/message_cloud.png'
def make_wc(self): with open('/home/suifeng/文档/text.txt', 'r') as f: text = f.read() text_spilt = ''.join(jieba.cut(text)) wc = WC( background_color='black', #背景色 max_words=100, #最大词数 max_font_size=80, #最大字体大小 random_state=35, # 最多的字样随机形态 font_path='MSYH.TTC' #字体路径,必须有字体,字体文件可以从Windows系统下复制过来 ).generate(text_spilt) plt.imshow(wc) plt.axis('off') plt.show() wc.to_file('/home/suifeng/图片/python词云图.png')
def wc(): # 读取小说内容 file = 'C:/Users/26015/Desktop/金瓶梅/金瓶梅.txt' with open(file, 'r', encoding='utf-8') as f: content = f.read() # 产生词云 wordcloud = WC( font_path='C:/Windows/Fonts/simkai.ttf' , width=1400 , height=700 ).generate(content) # plt.imshow(wordcloud) # plt.axis('off') # plt.show() # 保存图片 wordcloud.to_file("金瓶梅.jpg")
def __init__(self, movie, stop_words, srt_folder): self.movie = movie self.stop_words = stop_words self.subtitle = Subtitle.get_from_movie(movie, srt_folder) lines = [line.content for line in self.subtitle.get_lines()] self.words = tokenize_text(' '.join(lines)) self.wordcloud_title = f"{self.movie.original_title} ({self.movie.release_date})" self.filename = os.path.join(CONFIG['PNG_FOLDER'], f"{self.wordcloud_title}.png") self.cloud = WC(background_color="white", max_words=200, stopwords=set(self.stop_words), width=1280, height=720, collocations=False) # Related to issue_5: Duplicated words in word cloud. # With this parameter in False we avoid repeated words. self.cloud.generate(self.words) if not os.path.exists(CONFIG['PNG_FOLDER']): os.mkdir(CONFIG['PNG_FOLDER'])
def topic_wordcloud(self, figname: str = "wordcloud"): """ Uses WordCloud library to display the topic Use Laplace Smoothing and compare the vocabs from the date of interest with vocab from the date of the year before """ from wordcloud import WordCloud as WC import matplotlib.pyplot as plt prev_yr_voc = self._prev_voc prev_day_voc = self._voc.previous_day() prev_yr_voc.voc.update(prev_day_voc) self._voc = self.laplace_smoothing(self._voc, prev_yr_voc) # Create wordcloud wc = WC().generate_from_frequencies(self._voc) plt.imshow(wc) plt.axis('off') plt.tight_layout() plt.savefig(figname)
def __init__(self): self._min_word_len = 1 self._text_processors = [ replace_abbr_not(), # remove_len_less_than(3), sentence_tokenize(), ] self._sentence_processors = [ word_tokenize(), remove_stop_words(), ] self._words_processors = [ word_lemmatizing(), correct_spelling(), ] self._word_list = [] self._wc = WC( max_words=100, font_path=None, background_color='white', # width=400, height=200, mask=cv2.imread('word_cloud.mask0.png'))
def word_cloud_1(image,df1,number_clusters,selected_adjectives): for i in range(number_clusters): df_words=df1[df1['own_cluster']==i] df_words=df_words.adjectives flat_text=[item for sublist in df_words for item in sublist] text="" for word in flat_text: if word.lower() in selected_adjectives: text=text+" "+word wordcloud = WC(mask=image,background_color="white",contour_width=3, contour_color="black").generate(text) plt.imshow(wordcloud, interpolation='bilinear') plt.suptitle("Most numerous words in cluster".upper()) plt.title(str(i)) plt.axis("off") plt.show() return
def wc2(): # 读取小说内容 file = 'C:/Users/26015/Desktop/金瓶梅/金瓶梅.txt' with open(file, 'r', encoding='utf-8') as f: content = f.read() cut_content = jieba.cut(content.strip()) # 读取停用词 stopwords = stopwordslist('stopwords.txt') outstr = '' for word in cut_content: if word not in stopwords and word != '\t': outstr = outstr + word + " " # 产生词云 wordcloud = WC( font_path='C:/Windows/Fonts/simkai.ttf' , width=1400 , height=700 ).generate(outstr) # 保存图片 wordcloud.to_file("金瓶梅2.jpg")
def weibo_cloud(self): """ 微博分词 词云 :return: """ try: # 获取微博数据 data_count.main() # 读取数据文件 with open("./data/weibo/data-quting.txt", "r", encoding="utf-8") as f: wl_space_split = f.read() # print(wl_space_split) self.fig.clf() # 清除之前的画图 ax = self.fig.add_subplot(111) # 设置中文字体 font = r'C:\Windows\Fonts\simfang.ttf' # print("123456798") # 对分词后的文本生成词云 my_wordcloud = WC( collocations=False, font_path=font, width=3000, height=2000, background_color="white").generate(wl_space_split) # 显示词云图 ax.imshow(my_wordcloud) # 是否显示x轴、y轴下标 ax.axis("off") self.canvas.draw() # TODO:这里开始绘制 except Exception as e: print(e)
token.counter ## SHOW - number of times each bigram and word appear bigrams = {k: v for k, v in token.counter.items() if k.count("~")} cnt = Counter(bigrams) cnt.most_common(5) # [('of~the', 14615), # ('in~the', 9913), # ('to~the', 7339), # ('on~the', 4883), # ('and~the', 4843)] bigrams ## SHOW the selection # Word-cloud wc = WC().generate_from_frequencies(bigrams) plt.imshow(wc) plt.axis("off") plt.tight_layout() ## SHOW plt.savefig("books_wordcloud.png", dpi=300) ## SKIP # Histogram hist = defaultdict(list) _ = [hist[v].append(k) for k, v in bigrams.items()] plt.plot(np.log([len(hist.get(i, [0])) for i in range(1, 2000)])) plt.grid() plt.xlabel("Frequency") plt.ylabel("Number of bigrams (log-scale)") plt.tight_layout()
plt.plot(n, v, '.') plt.plot(n, k * n**beta) plt.legend(['Measured', 'Predicted']) plt.grid() plt.xlabel('Number of tokens') plt.ylabel('Vocabulary Size') plt.tight_layout() plt.savefig('heaps_law2.png', dpi=300) # Activities date = dict(year=2022, month=1, day=10) voc = Vocabulary(date, lang='Es', country='MX') words = {k: v for k, v in voc.voc.items() if not k.count('~')} wc = WC().generate_from_frequencies(words) plt.imshow(wc) plt.axis('off') plt.tight_layout() plt.savefig('wordcloud_mx.png', dpi=300) ## Zipf's Law - $$f=\frac{c}{r}$$ countries = [ 'MX', 'CO', 'ES', 'AR', 'PE', 'VE', 'CL', 'EC', 'GT', 'CU', 'BO', 'DO', 'HN', 'PY', 'SV', 'NI', 'CR', 'PA', 'UY' ] vocs = Parallel(n_jobs=-1)( delayed(Vocabulary)(date, lang='Es', country=country) for country in tqdm(countries)) words = [{k: v
words.insert(0, '<s>') words.append('</s>') _ = [(a, b) for a, b in zip(words, words[1:])] bigrams.update(_) words = set([a for a, b in bigrams]) words2 = set([b for a, b in bigrams]) prev = dict() for (a, b), v in bigrams.items(): try: prev[a] += v except KeyError: prev[a] = v wc = WC().generate_from_frequencies(prev) plt.imshow(wc) plt.axis('off') plt.tight_layout() P = defaultdict(Counter) for (a, b), v in bigrams.items(): next = P[a] next[b] = v / prev[a] wc = WC().generate_from_frequencies(P['<s>']) plt.imshow(wc) plt.axis('off') plt.tight_layout() plt.savefig('wordcloud_prob_start.png', dpi=300)
#! -*- coding: utf-8 -*- from scipy.misc import imread from wordcloud import ImageColorGenerator, WordCloud as WC text = open('files/python_en.txt').read() back_coloring = imread('python_logo.png') wc = WC(background_color='white', mask=back_coloring, random_state=42, margin=2) wc.generate(text) image_colors = ImageColorGenerator(back_coloring) wc.recolor(color_func=image_colors) plt.imshow(wc, interpolation='bilinear') plt.axis('off') plt.show() wc.to_file('image_colored_wordcloud.png')
#! -*- coding: utf-8 -*- import numpy as np from PIL import Image import matplotlib.pyplot as plt from wordcloud import WordCloud as WC text = open('files/python_en.txt').read() alice_mask = np.array(Image.open('logo.png')) wc = WC(background_color='white', mask=alice_mask, contour_width=2, contour_color='steelblue') wc.generate(text) plt.imshow(wc, interpolation='bilinear') plt.axis("off") plt.show() wc.to_file('masked_wordcloud.png')
X = tm.transform(_) m = LogisticRegression(multi_class='multinomial').fit(X, y[tr]) # m = LinearSVC().fit(X, y[tr]) _ = [D[x][0] for x in val] hy[val] = m.predict(tm.transform(_)) ci = bootstrap_confidence_interval(y, hy) ci D = list(tweet_iterator('../../../datasets/semeval/semeval2017_En_train.json')) tm = TextModel(token_list=[-1]).fit(D) id2word = {v:k for k, v in tm.model.word2id.items()} _ = {id2word[k]:v for k, v in tm.model.wordWeight.items()} wc = WC().generate_from_frequencies(_) plt.imshow(wc) plt.axis('off') plt.tight_layout() plt.savefig('semeval2017_idf.png', dpi=300) cnt = Counter() _ = [cnt.update(tm.tokenize(x)) for x in D] wc = WC().generate_from_frequencies(cnt) plt.imshow(wc) plt.axis('off') plt.tight_layout() plt.savefig('semeval2017_tf.png', dpi=300) perf = load_model('dataset/performance.gz')
#! -*- coding: utf-8 -*- import jieba from scipy.misc import imread from wordcloud import ImageColorGenerator, WordCloud as WC text = open('files/python_zh.txt').read() cn_text = ' '.join(jieba.cut(text)) font_path = '/usr/share/fonts/adobe-source-han-sans-cn/SourceHanSansCN-Normal.otf' back_coloring = imread('images/python_logo.png') wc = WC(font_path=font_path, background_color='white', mask=back_coloring, random_state=42, margin=2) wc.generate(cn_text) image_colors = ImageColorGenerator(back_coloring) wc.recolor(color_func=image_colors) plt.imshow(wc, interpolation='bilinear') plt.axis('off') plt.show() wc.to_file('images/wordcloud_cn.png')
okt = Okt() # 형태소 분석 객체 생성 kwds = okt.pos(x, norm=True, stem=True) kwds_filtered = [x for x, y in kwds if y in ['Noun', 'Adjective', 'Verb']] return kwds_filtered df_rev.loc[:, 'kwd'] = df_rev.loc[:, 'text'].map(get_kwds) df_rev.to_excel(writer, sheet_name='reviews') for topic in [topic_diff_max, topic_diff_min]: for model in ['A', 'B']: df_filtered = df_rev[(df_rev['model'] == model) & (df_rev['topic'] == topic)] list_kwd = df_filtered['kwd'].sum() dict_kwd_cnt = Counter(list_kwd) df_kwd_cnt = df({ 'kwd': list(dict_kwd_cnt.keys()), 'cnt': list(dict_kwd_cnt.values()) }) df_kwd_cnt = df_kwd_cnt.sort_values(by='cnt', ascending=False) sheet_name = f'kwd_cnt_{model}_{topic}' df_kwd_cnt.to_excel(writer, sheet_name=sheet_name, index=False) wc = WC(max_words=40, font_path='./font/SeoulNamsanM.ttf', background_color='white') wc.generate_from_frequencies(dict_kwd_cnt) rst_png_file_path = f'./sample/kwd_cnt_{model}_{topic}.png' wc.to_file(rst_png_file_path) writer.sheets[sheet_name].insert_image(3, 4, rst_png_file_path) writer.save()
_ = " & ".join(map(lambda x: "{:0.4f}".format(x), w)) print(r"{} \\".format(_)) R_m = W.sum(axis=1) C_m = W.sum(axis=0) ind = np.dot(np.atleast_2d(R_m).T, np.atleast_2d(C_m)) for w in (W - ind): _ = " & ".join(map(lambda x: "{:0.4f}".format(x), w)) print(r"{} \\".format(_)) # Example of the [bigrams](#tab:bivariate-distribution) _ = [(bigram, [index[x] for x in bigram.split("~")]) for bigram in bigrams] _ = {key: co_occurrence[i, j] for key, (i, j) in _} wc = WC().generate_from_frequencies(_) plt.imshow(wc) plt.axis('off') plt.tight_layout() plt.savefig('wordcloud_us.png', dpi=300) M = co_occurrence.sum(axis=1) def get_diff(key): a, b = [index[x] for x in key.split('~')] if a == b: return -M[a] * M[b] return co_occurrence[a, b] - M[a] * M[b]
#stemming the tweets from nltk.stem.porter import * stemmer = PorterStemmer() tokenized_tweet = tokenized_tweet.apply(lambda x: [stemmer.stem(i) for i in x]) #Rejoining the tweets combined['clean_tweet'] = tokenized_tweet.apply(lambda x: ' '.join(w for w in x)) words = ' '.join([word for word in combined['clean_tweet']]) # Generating the word cloud from wordcloud import WordCloud as WC cloud = WC(width=800,height=500,random_state=21,max_font_size=21).generate(words) plt.figure(figsize = (10,7)) plt.imshow(cloud,interpolation="bilinear") plt.axis('off') plt.show() # Generating normal words normal_words = ' '.join([word for word in combined['clean_tweet'][combined['label'] == 0]]) normal_wordcloud = WC(width=800,height=500,random_state=21,max_font_size=110).generate(normal_words) plt.figure(figsize = (10,7)) plt.imshow(normal_wordcloud,interpolation="bilinear") plt.axis('off') plt.show()
def review_analysis(src_file_path, rst_file_path, font_file_path='./font/SeoulNamsanM.ttf'): """ 구매 후기 데이터 처리 모듈 :param src_file_path: 구매 후기 원본 데이터 경로 :param rst_file_path: 처리 결과를 저장할 경로 :param font_file_path: 글꼴 경로 """ from pandas import DataFrame as df import re from statistics import mean from konlpy.tag import Okt from collections import Counter from wordcloud import WordCloud as WC f = open(src_file_path, 'rt', encoding='utf-8', errors='ignore') rev = f.read() f.close() ptrn_html_code = re.compile(r'&[a-zA-Z]+?;') # HTML 코드 패턴 ptrn_html_tag = re.compile(r'</?[a-zA-Z]+?>') # HTML 태그 패턴 ptrn_all_clean = re.compile(r'[^0-9a-zA-Z가-힣.,:/\s]') # 기타 불용어 패턴 for cleaner in [ptrn_html_code, ptrn_html_tag, ptrn_all_clean]: rev = re.sub(cleaner, '', rev) re_phrase = r'^([AB]):([a-z]+?)관련/점수:([1-5])점/내용:([0-9a-zA-Z가-힣.,:\s]+?)$' ptrn_phrase = re.compile(re_phrase, re.M) rev_values = re.findall(ptrn_phrase, rev) models, topics, scores, texts = zip(*rev_values) df_rev = df({ 'model': models, 'topic': topics, 'score': scores, 'text': texts }) df_rev['score'] = pd.to_numeric(df_rev['score'], downcast='integer', errors='ignore') df_rev_mean = df_rev.pivot_table(values='score', index='model', columns='topic', aggfunc=mean) df_rev_diff = df_rev_mean.diff(periods=-1) topic_diff_max = df_rev_diff.loc['A', :].idxmax() # B대비 A의 성과가 가장 좋은 토픽 탐색 topic_diff_min = df_rev_diff.loc['A', :].idxmin() # B대비 A의 성과가 가장 나쁜 토픽 탐색 writer = pd.ExcelWriter(rst_file_path, engine='xlsxwriter') # 엑셀 파일 객체 생성 df_rev_mean.to_excel(writer, sheet_name='score_mean') df_rev_diff.to_excel(writer, sheet_name='score_diff') m_chart = writer.book.add_chart({'type': 'radar'}) # 토픽별 평균 점수로 방사형 차트 생성 m_cat = ['score_mean', 0, 1, 0, 5] m_val_a = ['score_mean', 1, 1, 1, 5] m_val_b = ['score_mean', 2, 1, 2, 5] m_chart.add_series({'name': 'A', 'categories': m_cat, 'values': m_val_a}) m_chart.add_series({'name': 'B', 'categories': m_cat, 'values': m_val_b}) m_chart.set_title({'name': '토픽별 점수 평균'}) writer.sheets['score_mean'].insert_chart(3, 0, m_chart) d_chart = writer.book.add_chart({'type': 'column'}) # 토픽별 평균 점수 차이로 차트 생성 d_cat = ['score_diff', 0, 1, 0, 5] d_val = ['score_diff', 1, 1, 1, 5] d_chart.add_series({'name': 'A-B', 'categories': d_cat, 'values': d_val}) d_chart.set_title({'name': '토픽별 점수 평균 차이 (A-B)'}) writer.sheets['score_diff'].insert_chart(2, 0, d_chart) def get_kwds(x): okt = Okt() # 형태소 분석 객체 생성 kwds = okt.pos(x, norm=True, stem=True) kwds_filtered = [ x for x, y in kwds if y in ['Noun', 'Adjective', 'Verb'] ] return kwds_filtered df_rev.loc[:, 'kwd'] = df_rev.loc[:, 'text'].map(get_kwds) df_rev.to_excel(writer, sheet_name='reviews') for topic in [topic_diff_max, topic_diff_min]: for model in ['A', 'B']: df_filtered = df_rev[(df_rev['model'] == model) & (df_rev['topic'] == topic)] list_kwd = df_filtered['kwd'].sum() dict_kwd_cnt = Counter(list_kwd) df_kwd_cnt = df({ 'kwd': list(dict_kwd_cnt.keys()), 'cnt': list(dict_kwd_cnt.values()) }) df_kwd_cnt = df_kwd_cnt.sort_values(by='cnt', ascending=False) sheet_name = f'kwd_cnt_{model}_{topic}' df_kwd_cnt.to_excel(writer, sheet_name=sheet_name, index=False) wc = WC(max_words=40, font_path=font_file_path, background_color='white') wc.generate_from_frequencies(dict_kwd_cnt) rst_png_file_path = f'./sample/kwd_cnt_{model}_{topic}.png' wc.to_file(rst_png_file_path) writer.sheets[sheet_name].insert_image(3, 4, rst_png_file_path) writer.save()
token.counter ## SHOW - number of times each bigram and word appear bigrams = {k: v for k, v in token.counter.items() if k.count("~")} cnt = Counter(bigrams) cnt.most_common(5) # [('of~the', 14615), # ('in~the', 9913), # ('to~the', 7339), # ('on~the', 4883), # ('and~the', 4843)] bigrams ## SHOW the selection # Word-cloud wc = WC().generate_from_frequencies(bigrams) plt.imshow(wc) plt.axis("off") plt.tight_layout() ## SHOW plt.savefig("books_wordcloud.png", dpi=300) ## SKIP # Histogram hist = defaultdict(list) _ = [hist[v].append(k) for k, v in bigrams.items()] plt.plot(np.log([len(hist.get(i, [0])) for i in range(1, 2000)])) plt.grid() plt.xlabel("Frequency") plt.ylabel("Number of bigrams (log-scale)") plt.tight_layout()