class deal_cache_user_info(threading.Thread): def __init__(self): threading.Thread.__init__(self) self.dbi = MySQL_Interface() self.bf = BloomFilter() def run(self): while True: if self.dbi.is_empty('cache_user_info'): time.sleep(2) self.dbi = MySQL_Interface() continue [res, cache_user_info_col] = self.dbi.select_all('cache_user_info') time_stick = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) # insert into user info table user_info_table_col = self.dbi.get_col_name('user_info_table') data = [[ line[cache_user_info_col.index(col)] if col in cache_user_info_col else time_stick if col == 'insert_time' else None if col == 'update_time' else None if col == 'latest_blog' else None if col == 'isGettingBlog' else '' for col in user_info_table_col ] for line in res] uid_list = [ line[user_info_table_col.index('uid')] for line in data ] self.dbi.insert_asList('user_info_table', data, unique=True) # 插入 user info table self.bf.insert_asList(uid_list, 'user_info_table') print('insert {num} users into user info table'.format( num=data.__len__())) uid_list = [line[cache_user_info_col.index('uid')] for line in res] q1 = "delete from {table_name} where uid in ( {id_str_list} ) ;" # 从cache user info 中删除 id_str_list = '' for i in uid_list: id_str_list = id_str_list + '\'' + str(i) + '\'' + ',' id_str_list = id_str_list[:-1] query = q1.format(id_str_list=id_str_list, table_name='cache_user_info') self.dbi.cur.execute(query) self.dbi.conn.commit() query = q1.format(id_str_list=id_str_list, table_name='ready_to_get') self.dbi.cur.execute(query) self.dbi.conn.commit()
class deal_cache_user_info(threading.Thread): def __init__(self): threading.Thread.__init__(self) self.dbi=MySQL_Interface() self.bf=BloomFilter() def run(self): while True: if self.dbi.is_empty('cache_user_info'): time.sleep(2) self.dbi=MySQL_Interface() continue [res,cache_user_info_col]=self.dbi.select_all('cache_user_info') time_stick=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) # insert into user info table user_info_table_col=self.dbi.get_col_name('user_info_table') data= [ [ line[cache_user_info_col.index(col)] if col in cache_user_info_col else time_stick if col=='insert_time' else None if col=='update_time' else None if col=='latest_blog' else None if col=='isGettingBlog' else '' for col in user_info_table_col ] for line in res] uid_list=[line[user_info_table_col.index('uid')] for line in data] self.dbi.insert_asList('user_info_table',data,unique=True) # 插入 user info table self.bf.insert_asList(uid_list,'user_info_table') print('insert {num} users into user info table'.format(num=data.__len__())) uid_list=[line[cache_user_info_col.index('uid')] for line in res] q1="delete from {table_name} where uid in ( {id_str_list} ) ;" # 从cache user info 中删除 id_str_list='' for i in uid_list: id_str_list=id_str_list+'\''+str(i)+'\''+',' id_str_list=id_str_list[:-1] query=q1.format(id_str_list=id_str_list,table_name='cache_user_info') self.dbi.cur.execute(query) self.dbi.conn.commit() query=q1.format(id_str_list=id_str_list,table_name='ready_to_get') self.dbi.cur.execute(query) self.dbi.conn.commit()
__author__ = 'multiangle' from DB_Interface import MySQL_Interface import json import networkx as nx dbi=MySQL_Interface() # create table (select * from user_info_table order by fans_num limit 1000) [web_info,col_info]=dbi.select_all('temp_table2') select_web=[] select_user={} for atte in web_info: if (atte[1],atte[0]) in web_info: select_web.append(list(atte)) select_user[atte[1]]=1 select_user[atte[0]]=1 select_user=select_user.keys() G=nx.Graph() G.add_nodes_from(select_user) G.add_edges_from(select_web) nx.write_gexf(G,'weibo_node1000.gexf')
def test2(): from DB_Interface import MySQL_Interface dbi = MySQL_Interface() [x, s] = dbi.select_all("ready_to_get") print(x)
__author__ = 'multiangle' from DB_Interface import MySQL_Interface import json import networkx as nx dbi = MySQL_Interface() # create table (select * from user_info_table order by fans_num limit 1000) [web_info, col_info] = dbi.select_all('temp_table2') select_web = [] select_user = {} for atte in web_info: if (atte[1], atte[0]) in web_info: select_web.append(list(atte)) select_user[atte[1]] = 1 select_user[atte[0]] = 1 select_user = select_user.keys() G = nx.Graph() G.add_nodes_from(select_user) G.add_edges_from(select_web) nx.write_gexf(G, 'weibo_node1000.gexf')
__author__ = 'multiangle' import networkx as nx import matplotlib.pyplot as plt import numpy as np from DB_Interface import MySQL_Interface dbi = MySQL_Interface() [select_user, select_user_col] = dbi.select_all('select_user') user_list = [line[select_user_col.index('name')] for line in select_user] user_id = [line[select_user_col.index('uid')] for line in select_user] [atten_web, atten_web_col] = dbi.select_all('select_atten') atten_list = [[ line[atten_web_col.index('from_uid')], line[atten_web_col.index('to_uid')] ] for line in atten_web] # temp_atten_list=[] # for line in atten_list: # try: # temp=[user_list[user_id.index(line[0])],user_list[user_id.index(line[1])]] # temp_atten_list.append(temp) # except: # pass # atten_list=temp_atten_list print(atten_list.__len__()) sig_list = [line[0] + line[1] for line in atten_list] select_atten_list = [] for line in atten_list: temp_sig_a = line[0] + line[1] temp_sig_b = line[1] + line[0]
__author__ = 'multiangle' import networkx as nx import matplotlib.pyplot as plt import numpy as np from DB_Interface import MySQL_Interface dbi=MySQL_Interface() [select_user,select_user_col]=dbi.select_all('select_user') user_list= [line[select_user_col.index('name')] for line in select_user] user_id=[line[select_user_col.index('uid')] for line in select_user] [atten_web,atten_web_col]=dbi.select_all('select_atten') atten_list=[[line[atten_web_col.index('from_uid')],line[atten_web_col.index('to_uid')]] for line in atten_web] # temp_atten_list=[] # for line in atten_list: # try: # temp=[user_list[user_id.index(line[0])],user_list[user_id.index(line[1])]] # temp_atten_list.append(temp) # except: # pass # atten_list=temp_atten_list print(atten_list.__len__()) sig_list= [line[0]+line[1] for line in atten_list] select_atten_list=[] for line in atten_list: temp_sig_a=line[0]+line[1] temp_sig_b=line[1]+line[0] if temp_sig_a in sig_list and temp_sig_b in sig_list : select_atten_list.append(line)