def getWeiboIds():
weiboids = []
myconnect = GetConnect()
sql = 'select weiboid from profile where is_education = -1'
results = myconnect.getData(sql)
if results:
for r in results:
weiboids.append(r[0])
return weiboids
def getWeiboIds(schooltable): weiboids = [] myconnect = GetConnect() sql = 'select weiboid from %s where is_profile = -1' % schooltable results = myconnect.getData(sql) if results: for r in results: weiboids.append(r[0]) return weiboids
def getWeiboIds(): weiboids = [] myconnect = GetConnect() sql = 'select weiboid from profile where is_education = -1' results = myconnect.getData(sql) if results: for r in results: weiboids.append(r[0]) return weiboids
def get_school_weibo(schoolname):
myconnect = GetConnect()
school_weibo_table = schoolname + '_wordsegment'
school_weibo = 'select segments, is_meaningful from %s' % school_weibo_table
weibo_content_results = myconnect.getData(school_weibo)
print len(weibo_content_results)
for i in xrange(100):
print weibo_content_results[i][0],weibo_content_results[i][1]
pickle.dump(weibo_content_results, open('dict weibo data\\%s_seg_weibo.pkl' % (schoolname), 'w'))
def getSchooWeiboMeaning(schoolname): schoolname = schoolname + '_wordsegment' countMeaning = [] myconnect = GetConnect() get_school_weibo_meaning_num = 'select count(*) as meaningcount,is_meaningful from %s group by is_meaningful;' % schoolname meaning_num_results = myconnect.getData(get_school_weibo_meaning_num) if meaning_num_results: countMeaning.append(int(meaning_num_results[1][0])) countMeaning.append(int(meaning_num_results[0][0])) return countMeaning
def getWeiboIds(schoolname=None): '获取schoolname表中没有下载原创无图微博的用户' weiboids = [] sql = "select weiboid from %s order by rand() limit 20" % schoolname myconnect = GetConnect() results = myconnect.getData(sql) if results: for r in results: weiboids.append(r[0]) #print r[0] return weiboids
def getWeiboIds(schoolname=None):
'获取schoolname表中没有下载原创无图微博的用户'
weiboids = []
sql = "select weiboid from %s order by rand() limit 20" % schoolname
myconnect = GetConnect()
results = myconnect.getData(sql)
if results:
for r in results:
weiboids.append(r[0])
#print r[0]
return weiboids
def get_one_weibo_data(schoolname, weiboid=None): myconnect = GetConnect() if weiboid is None: get_weibo_id_sql = "select weiboid from %s where is_wb_ori_no_pic = 1 order by rand() limit 1;" % schoolname results = myconnect.getData(get_weibo_id_sql) if results: weiboid = results[0][0] else: print "get weiboid wrong" weiboid = '2591961830' if schoolname == 'dlut': school_weibo_table = 'wb_ori_no_pic' else: school_weibo_table = schoolname + '_wb_ori_no_pic' get_weibo_content_sql = "select content, upvotes, forwards, reviews, weiboid from %s where weiboid = %s" % (school_weibo_table, weiboid) weibo_content_results = myconnect.getData(get_weibo_content_sql) if weibo_content_results: return weibo_content_results else: return None
def getMarkedWeibo():
marked_weibo_data = []
pos_weibo = [] # 表达积极情绪的微博
neg_weibo = [] # 表达消极情绪的微博
act_weibo = [] # 代表参与一个活动随手转发的微博，一般都是广告，含有链接
obj_weibo = [] # 客观的微博，即不包含任何感情的微博
# Read txt file contain sentiment stopwords
sentiment_stopwords = get_txt_data('sentiment_stopword.txt', 'lines')
# 获取已经标注过的微博数据，-1代表未知，1代表积极，2代表消极，
# 3代表活动（有链接，一般是广告），4是客观（没有表达任何情绪）
get_mark_weibo_sql = "select content, mark from markedweibo;"
myconnect = GetConnect()
results = myconnect.getData(get_mark_weibo_sql)
for weibo in results:
if weibo[1] == 1:
pos_weibo.append(weibo[0])
elif weibo[1] == 2:
neg_weibo.append(weibo[0])
elif weibo[1] == 3:
act_weibo.append(weibo[0])
elif weibo[1] == 4:
obj_weibo.append(weibo[0])
# Filter stopwords from reviews
seg_pos_result = []
seg_neg_result = []
seg_act_result = []
seg_obj_result = []
for weibo in pos_weibo:
fil = [word for word in weibo if word not in sentiment_stopwords and word != ' ']
seg_pos_result.append(fil)
fil = []
for weibo in neg_weibo:
fil = [word for word in weibo if word not in sentiment_stopwords and word != ' ']
seg_neg_result.append(fil)
fil = []
for weibo in act_weibo:
fil = [word for word in weibo if word not in sentiment_stopwords and word != ' ']
seg_act_result.append(fil)
fil = []
for weibo in obj_weibo:
fil = [word for word in weibo if word not in sentiment_stopwords and word != ' ']
seg_obj_result.append(fil)
fil = []
# 将这些数据存储成pickle
pickle.dump(pos_weibo, open("maked weibo\\seg_pos_result.pkl", 'w'))
pickle.dump(neg_weibo, open("maked weibo\\seg_neg_result.pkl", 'w'))
pickle.dump(act_weibo, open("maked weibo\\seg_act_result.pkl", 'w'))
pickle.dump(obj_weibo, open("maked weibo\\seg_obj_result.pkl", 'w'))
def conveyToSchoolTable(schoolname, schooltable): '将education表中所有大学的微博用户添加到表中' global GetConnect sql = "select * from education where school = '%s'" % schoolname myconnect = GetConnect() results = myconnect.getData(sql) school_d = School_Db(schooltable) if results: for r in results: schoolers = School_Info(r[1]) school_d.insertIntoDB(schoolers) #print r[1] countsql = "select * from %s" % schooltable count = myconnect.getCount(countsql) return count
def conveyToSchoolTable(schoolname): '将education表中所有大连理工大学的微博用户添加到dlut表中' global GetConnect sql = "select * from education where school = '%s'" % schoolname myconnect = GetConnect() results = myconnect.getData(sql) dlut_d = Dlut_Db() if results: for r in results: dluters = Dlut(r[1]) dlut_d.insertIntoDB(dluters) #print r[1] countsql = "select * from dlut" count = myconnect.getCount(countsql) return count
def conveyToSchoolTable(schoolname, schooltable):
'将education表中所有大学的微博用户添加到表中'
global GetConnect
sql = "select * from education where school = '%s'" % schoolname
myconnect = GetConnect()
results = myconnect.getData(sql)
school_d = School_Db(schooltable)
if results:
for r in results:
schoolers = School_Info(r[1])
school_d.insertIntoDB(schoolers)
#print r[1]
countsql = "select * from %s" % schooltable
count = myconnect.getCount(countsql)
return count
def get_school_weibo_and_save(schoolname):
if schoolname == 'dlut':
schooltable = 'wb_ori_no_pic'
else:
schooltable = schoolname + '_wb_ori_no_pic'
get_weibo_sql = "select content from %s;" % schooltable
myconnect = GetConnect()
results = myconnect.getData(get_weibo_sql)
school_weibo = []
index = 0
for i in results:
school_weibo.append(i[0])
if index < 100:
print i[0].encode('utf-8')
index += 1
pickle.dump(school_weibo, open("machine learning data\\%s_weibo.pkl" % schoolname,'w'))
def getWordFrequency(schoolname):
get_keyword_sql = 'select keywords from %s where is_meaningful = 1' % (schoolname+'_wordsegment')
myconnect = GetConnect()
results = myconnect.getData(get_keyword_sql)
worddict = {} # 单词字典，记录所有出现的单词以及出现的次数
print len(results)
index = 0
for r in results:
for w in r[0].split():
index += 1
if worddict.has_key(w) == False:
worddict[w] = 1
else:
worddict[w] += 1
print index, 'index' # 这是单词出现的总数 dlut有27万词
print len(worddict) # 这是存储到字典中的单词总数，dlut有6.5万词
pickle.dump(worddict, open('weibo word contrast\\%s_worddict.pkl' % schoolname, 'w'))
def getWordFrequency(schoolname):
get_keyword_sql = 'select keywords from %s where is_meaningful = 1' % (
schoolname + '_wordsegment')
myconnect = GetConnect()
results = myconnect.getData(get_keyword_sql)
worddict = {} # 单词字典，记录所有出现的单词以及出现的次数
print len(results)
index = 0
for r in results:
for w in r[0].split():
index += 1
if worddict.has_key(w) == False:
worddict[w] = 1
else:
worddict[w] += 1
print index, 'index' # 这是单词出现的总数 dlut有27万词
print len(worddict) # 这是存储到字典中的单词总数，dlut有6.5万词
pickle.dump(worddict,
open('weibo word contrast\\%s_worddict.pkl' % schoolname, 'w'))
