forked from ayizhi/weiboScrapy2.0_py
/
analysis.py
75 lines (58 loc) · 1.57 KB
/
analysis.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
# !/usr/bin/python
# coding=utf-8
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import jieba
import sqlite3 as sqlite
import time
import datetime
import clusters
from PIL import Image,ImageDraw
con = sqlite.connect('./data/myWeibo.db')
allData = con.execute('select * from myWeibodata').fetchall()
# time1 = datetime.datetime.strptime(allData[0][0],'%Y-%m-%d %H:%M')
# print time1#得到的字符串
# print time.mktime(time1.timetuple())#转化为时间戳
def makeDate(allData):
allInfo = {}
words = {}
wordneed = []
times = []
for i in range(0,len(allData)):
date = datetime.datetime.strptime(allData[i][0],'%Y-%m-%d %H:%M')
dateTuple = time.mktime(date.timetuple())
times.append(dateTuple)
content = ''
if allData[i][2] == 0:#转发
content = allData[i][3]
else:
content = allData[i][1]
jiebaContent = jieba.lcut(content,cut_all = False)#分词
obj = {}
# print jiebaContent
for r in range(0,len(jiebaContent)):
words[jiebaContent[r]] = 0
obj.setdefault(jiebaContent[r],0)
obj[jiebaContent[r]] = 1 if obj[jiebaContent[r]] == 0 else obj[jiebaContent[r]] + 1
allInfo[dateTuple] = obj
dataList = []
for i in words:
wordneed.append(i)
for r in allInfo:
print r
clone = words
theList = []
for i in allInfo[r]:
if i in words:
clone[i] = allInfo[r][i]
for n in clone:
theList.append(clone[n])
dataList.append(theList)
return dataList,times,wordneed
data,time,word = makeDate(allData)
print 'done'
# print len(data)
clust = clusters.hcluster(data)
print 'done'
clusters.drawdendrogram(clust,time,jpeg = './myWeibo.jpg')