class deal_isGettingBLog_user(threading.Thread): def __init__(self): threading.Thread.__init__(self) self.dbi=MySQL_Interface() def run(self): while True: self.dbi=MySQL_Interface() t=time.time() time_stick=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(t-12*60*60)) #删掉cache_history中的行 query='delete from cache_history where container_id in (select container_id from user_info_table where isGettingBlog<\'{time}\' and update_time is null)'\ .format(time=time_stick) self.dbi.update_asQuery(query) # 删掉mongodb-assemble factory中的相关值 select_query='select container_id from user_info_table where isGettingBlog<\'{time}\' and update_time is null'.format(time=time_stick) res=[x[0] for x in self.dbi.select_asQuery(select_query)] client=MongoClient('localhost',27017) db=client['microblog_spider'] assemble_table=db.assemble_factory assemble_table.remove({'container_id':{'$in':res}}) # 将user info table中超时行的isGettingBlog清空 query="update user_info_table set isGettingBlog=null where isGettingBlog<\'{time}\' and update_time is null".format(time=time_stick) self.dbi.update_asQuery(query) # 将cache_history中的残留项去除 query = "delete from cache_history where is_dealing<\'{time}\' ;".format(time = time_stick) self.dbi.update_asQuery(query) time.sleep(60)
class deal_isGettingBLog_user(threading.Thread): def __init__(self): threading.Thread.__init__(self) self.dbi=MySQL_Interface() def run(self): while True: self.dbi=MySQL_Interface() t=time.time() time_stick=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(t-12*60*60)) #删掉cache_history中的行 query='delete from cache_history where container_id in (select container_id from user_info_table where isGettingBlog<\'{time}\' and update_time is null)'\ .format(time=time_stick) self.dbi.update_asQuery(query) # 删掉mongodb-assemble factory中的相关值 select_query='select container_id from user_info_table where isGettingBlog<\'{time}\' and update_time is null'.format(time=time_stick) res=[x[0] for x in self.dbi.select_asQuery(select_query)] client=MongoClient('localhost',27017) db=client['microblog_spider'] assemble_table=db.assemble_factory assemble_table.remove({'container_id':{'$in':res}}) # 将user info table中超时行的isGettingBlog清空 query="update user_info_table set isGettingBlog=null where isGettingBlog<\'{time}\' and update_time is null".format(time=time_stick) self.dbi.update_asQuery(query) time.sleep(60)
class deal_cache_attends(threading.Thread): def __init__(self): threading.Thread.__init__(self) dbi = MySQL_Interface() self.dbi = dbi self.bf = BloomFilter() def run(self): bag = [] uid_bag = [] #与bag类似,只不过存储uid bag_size = 1000 #100次插入一次 ready_to_get_col = self.dbi.get_col_name('ready_to_get') cache_attends_col = self.dbi.get_col_name('cache_attends') while True: query = 'select * from cache_attends limit 5000' res = self.dbi.select_asQuery(query) if res.__len__() == 0: if bag.__len__() > 0: self.dbi.insert_asList('ready_to_get', bag, unique=True) bag = [] # self.bf.insert_asList(uid_bag,'ready_to_get') uid_bag = [] time.sleep(1) self.dbi = MySQL_Interface() #更新dbi continue print('thread cache attends is working') for line in res: raw_id = line[cache_attends_col.index('uid')] in_user_info = self.bf.isContains(raw_id, 'user_info_table') #此处可优化 if not in_user_info: data = [ line[cache_attends_col.index(col)] if col in cache_attends_col else None for col in ready_to_get_col ] bag.append(data) uid_bag.append(raw_id) if bag.__len__() > bag_size: self.dbi.insert_asList('ready_to_get', bag, unique=True) # self.bf.insert_asList(uid_bag,'ready_to_get') print('insert once') bag = [] uid_bag = [] self.dbi.delete_line('cache_attends', 'uid', raw_id) # 此处可优化 def isInUserInfo(self, in_uid): col_user_info = self.dbi.get_col_name('user_info_table') query = 'select * from user_info_table where uid={uid}'.format( uid=in_uid) res = self.dbi.select_asQuery(query) if res.__len__() == 0: return False else: return True
class deal_cache_attends(threading.Thread): def __init__(self): threading.Thread.__init__(self) dbi=MySQL_Interface() self.dbi=dbi self.bf=BloomFilter() def run(self): bag=[] uid_bag=[] #与bag类似,只不过存储uid bag_size=1000 #100次插入一次 ready_to_get_col=self.dbi.get_col_name('ready_to_get') cache_attends_col=self.dbi.get_col_name('cache_attends') while True: query='select * from cache_attends limit 5000' res=self.dbi.select_asQuery(query) if res.__len__()==0: if bag.__len__()>0: self.dbi.insert_asList('ready_to_get',bag,unique=True) bag=[] # self.bf.insert_asList(uid_bag,'ready_to_get') uid_bag=[] time.sleep(1) self.dbi=MySQL_Interface() #更新dbi continue print('thread cache attends is working') for line in res: raw_id=line[cache_attends_col.index('uid')] in_user_info=self.bf.isContains(raw_id,'user_info_table') #此处可优化 if not in_user_info: data=[line[cache_attends_col.index(col)] if col in cache_attends_col else None for col in ready_to_get_col] bag.append(data) uid_bag.append(raw_id) if bag.__len__()>bag_size: self.dbi.insert_asList('ready_to_get',bag,unique=True) # self.bf.insert_asList(uid_bag,'ready_to_get') print('insert once') bag=[] uid_bag=[] self.dbi.delete_line('cache_attends','uid',raw_id) # 此处可优化 def isInUserInfo(self,in_uid): col_user_info=self.dbi.get_col_name('user_info_table') query='select * from user_info_table where uid={uid}'.format(uid=in_uid) res=self.dbi.select_asQuery(query) if res.__len__()==0: return False else: return True
class control_ready_table(threading.Thread): def __init__(self): threading.Thread.__init__(self) self.dbi=MySQL_Interface() def run(self): while True: self.dbi=MySQL_Interface() num=self.dbi.get_line_num('ready_to_get') if num>150*1000: query='select m.fans_num from (' \ 'select fans_num from ready_to_get ' \ 'ORDER BY fans_num limit 50000' \ ') as m order by fans_num desc limit 1' res=self.dbi.select_asQuery(query)[0][0] query='delete from ready_to_get where fans_num<{num}'\ .format(num=res) self.dbi.update_asQuery(query) else: time.sleep(600)
def get(self): global proxy uuid=str(self.get_argument('uuid')) task_id=self.task_assign(uuid) if proxy.get_ave_proxy_size()<30: # check the size of current proxy size self.write('no task') self.finish() return if task_id==-1: # checi if this uuid is valid self.write('no task') self.finish() return if task_id==1: # get the social web of certain user dbi=MySQL_Interface() query='select * from ready_to_get where is_fetching is null order by fans_num desc limit 1;' res=dbi.select_asQuery(query) if res.__len__()==0: self.write('no task') self.finish() return res=res[0] col_info=dbi.get_col_name('ready_to_get') uid=res[col_info.index('uid')] self.write('{uid},connect'.format(uid=uid)) self.finish() time_stick=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) query="update ready_to_get set is_fetching=\'{t_time}\' where uid={uid} ;"\ .format(t_time=time_stick,uid=uid) dbi.update_asQuery(query) if task_id==2: # get the history microblog of a certain user dbi=MySQL_Interface() query='select container_id,blog_num from user_info_table ' \ 'where (isGettingBlog is null and update_time is null) ' \ 'order by fans_num desc limit 1 ;' # query='select container_id,blog_num from user_info_table ' \ # 'order by rand() limit 1 ;' [container_id,blog_num]=dbi.select_asQuery(query)[0] self.write('{c_id};{blog},history' .format(c_id=container_id,blog=blog_num)) self.finish() time_stick=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) query="update user_info_table set isGettingBlog=\'{t_time}\' where container_id={cid} ;"\ .format(t_time=time_stick,cid=container_id) dbi.update_asQuery(query) if task_id==3: # this part is in test dbi=MySQL_Interface() current_time_stick=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) target_time_stick=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()-60*60*24)) #提早一天 query='select uid,update_time from user_info_table ' \ 'where update_time<\'{target_time}\' and isGettingBlog is null limit {batch}'\ .format(target_time=target_time_stick,batch=10) res=dbi.select_asQuery(query) res=[[line[0],int(time.mktime(line[1].timetuple()))] for line in res] # res=[[line[0],int(time.mktime(time.strptime(str(line[1]),'%Y-%m-%d %H:%M:%S')))] for line in res] res=[line[0]+'-'+str(line[1]) for line in res] inn='' for item in res: inn+=item+';' inn=inn[0:-1] # uid-stamp;uid-timestamp;...;,update (the formation of order) commend='{list},update'.format(list=inn) self.write(commend) self.finish()
def post(self): # 从客户端获取信息 try: user_history=self.get_argument('user_history') latest_time=self.get_argument('latest_time') latest_timestamp=self.get_argument('latest_timestamp') container_id=self.get_argument('container_id') isDivided=self.get_argument('isDivided') user_history=eval(user_history) if isDivided==1 or isDivided=='1' : block_num=self.get_argument('block_num') current_block=self.get_argument('current_block') self.write('success to return user history') self.finish() print('Success: to get data from web') except Exception as e: self.write('fail to return user history') self.finish() print('Error:server-HistoryReturn:' 'Unable to get value from http package,Reason:') print(e) return # 连接 try: dbi=MySQL_Interface() except: print('Error:server-HistoryReturn:' 'Unable to connect to MySQL') # 从MYSQL获取该用户相关信息 try: query='select * from user_info_table where container_id=\'{cid}\''\ .format(cid=container_id) user_info=dbi.select_asQuery(query)[0] col_name=dbi.get_col_name('user_info_table') except Exception as e: print('Error:server-HistoryReturn:' 'No such user in MySQL.user_info_table,Reason:') print(e) # 将数据存入Mongodb以后将相关信息存入mysql,并将isGettingBlog字段设为空 try: blog_len=user_history.__len__() wanted_blog_len=user_info[col_name.index('blog_num')] blog_accuracy=blog_len/wanted_blog_len time_stick=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) if not user_info[col_name.index('update_time')]: save_data_inMongo(user_history) # query='update user_info_table set ' \ # 'update_time=\'{up_time}\',' \ # 'latest_blog=\'{latest_blog}\',' \ # 'isGettingBlog=null ' \ # 'where container_id=\'{cid}\';'\ # .format(up_time=time_stick,latest_blog=latest_time,cid=container_id) query='update user_info_table set ' \ 'update_time=\'{up_time}\',' \ 'latest_blog=\'{latest_blog}\'' \ 'where container_id=\'{cid}\';' \ .format(up_time=time_stick,latest_blog=latest_time,cid=container_id) dbi.update_asQuery(query) else: query='update user_info_table set isGettingBlog=null where container_id=\'{cid}\''\ .format(cid=container_id) dbi.update_asQuery(query) query='insert into accuracy_table values ({acc},\'{t_s}\') ;'\ .format(acc=blog_accuracy,t_s=time_stick) dbi.insert_asQuery(query) print('Success: insert user into MongoDB, the num of data is {len}' .format(len=blog_len)) except Exception as e: print('Error:server-HistoryReturn:' 'Unable to update data in MySQL.user_info_tabe,Reason:') print(e)
def get(self): global proxy uuid=str(self.get_argument('uuid')) task_id=self.task_assign(uuid) if proxy.get_ave_proxy_size()<30: # check the size of current proxy size self.write('no task') self.finish() return if task_id==-1: # checi if this uuid is valid self.write('no task') self.finish() return if task_id==1: # get the social web of certain user dbi=MySQL_Interface() query='select * from ready_to_get where is_fetching is null order by fans_num desc limit 1;' res=dbi.select_asQuery(query) if res.__len__()==0: self.write('no task') self.finish() return res=res[0] col_info=dbi.get_col_name('ready_to_get') uid=res[col_info.index('uid')] self.write('{uid},connect'.format(uid=uid)) self.finish() time_stick=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) query="update ready_to_get set is_fetching=\'{t_time}\' where uid={uid} ;"\ .format(t_time=time_stick,uid=uid) dbi.update_asQuery(query) if task_id==2: # get the history microblog of a certain user dbi=MySQL_Interface() query='select container_id,blog_num from user_info_table ' \ 'where (isGettingBlog is null and update_time is null and blog_num<{valve} and blog_num>100)' \ 'order by fans_num desc limit 1 ;'.format(valve=config.HISTORY_TASK_VALVE) # query='select container_id,blog_num from user_info_table ' \ # 'order by rand() limit 1 ;' res=dbi.select_asQuery(query) if res.__len__()==0: self.write('no task') self.finish() return [container_id,blog_num]=res[0] self.write('{c_id};{blog},history' .format(c_id=container_id,blog=blog_num)) self.finish() time_stick=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) query="update user_info_table set isGettingBlog=\'{t_time}\' where container_id={cid} ;"\ .format(t_time=time_stick,cid=container_id) dbi.update_asQuery(query) if task_id==3: # get the history microblog of a certain user dbi=MySQL_Interface() query='select container_id,blog_num from user_info_table ' \ 'where (isGettingBlog is null and update_time is null and blog_num>={valve} and blog_num>100)' \ 'order by fans_num desc limit 1 ;'.format(valve=config.HISTORY_TASK_VALVE) # query='select container_id,blog_num from user_info_table ' \ # 'order by rand() limit 1 ;' [container_id,blog_num]=dbi.select_asQuery(query)[0] self.write('{c_id};{blog},history' .format(c_id=container_id,blog=blog_num)) self.finish() time_stick=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) query="update user_info_table set isGettingBlog=\'{t_time}\' where container_id={cid} ;" \ .format(t_time=time_stick,cid=container_id) dbi.update_asQuery(query) if task_id==4 or task_id==5 or task_id==100: # this part is in test dbi=MySQL_Interface() current_time_stick=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) target_time_stick=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()-60*60*24*1)) #提早5天 if task_id==4: batch_size = 100 elif task_id==5: batch_size = 200 else: batch_size = 10 query='select container_id,update_time,latest_blog from user_info_table ' \ 'where update_time<\'{target_time}\' and isGettingBlog is null and blog_num>10 order by fans_num desc limit {batch}' \ .format(target_time=target_time_stick,batch=batch_size) print(query) res=dbi.select_asQuery(query) # 将从mysql中取得的用户列表加上必要的变量以后发送给客户端 res=[[line[0],int(time.mktime(line[1].timetuple())),int(time.mktime(line[2].timetuple()))] for line in res] res_cp=res if res_cp.__len__()==0: # if no task ,then return "no task" print('*** warning: no avaliable update mission ***') self.write('no task') self.finish() return # print('debug from task handler') # pprint(res_cp) res=[line[0]+'-'+str(line[1])+'-'+str(line[2]) for line in res] inn='' for item in res: inn+=item+';' inn=inn[0:-1] # uid-stamp;uid-timestamp;...;,update (the formation of order) mission_id=random_str(15) commend='{list};{task_id},update'.format(list=inn,task_id=mission_id) # 传送给客户端的指令格式: ContainerId-UpdateTime-LatestBlog;...;...;...,update self.write(commend) self.finish() # 将用户列表,任务id,以及任务开始时间存入mongodb u_list=[dict(container_id=x[0],update_time=x[1],latest_blog=x[2]) for x in res_cp] data_toMongo=dict( mission_id = mission_id, user_list = u_list, mission_start= int(time.time()) ) client=MongoClient('localhost',27017) db=client['microblog_spider'] collec=db.update_mission collec.insert(data_toMongo) # 将相关内容从mysql中设置isGettingBlog user_list_str='' for line in res_cp: user_list_str+='\'{cid}\','.format(cid=line[0]) user_list_str=user_list_str[:-1] time_stick=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) query='update user_info_table set isGettingBlog=\'{time}\' where container_id in ({ulist})'\ .format(time=time_stick,ulist=user_list_str) dbi.update_asQuery(query)
def run(self): while True: start_time = time.time() dbi = MySQL_Interface() col_info = dbi.get_col_name('cache_history') query = 'select * from cache_history where is_dealing is null order by checkin_timestamp limit 1' mysql_res = dbi.select_asQuery(query) if mysql_res.__len__() == 0: # cache_history表为空时,睡眠1秒,跳过此次循环 time.sleep(1) continue mysql_res = mysql_res[0] # todo for delete----- print('debug->start to deal with a new task') print('debug->mysql_res: ') print(mysql_res) #------------------------ container_id = mysql_res[col_info.index('container_id')] print('debug->container_id: {cid}'.format(cid=container_id)) latest_time = mysql_res[col_info.index('latest_time')] latest_timestamp = mysql_res[col_info.index('latest_timestamp')] time_stick = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) query = 'update cache_history set is_dealing=\'{time}\' where container_id={cid}'.format( time=time_stick, cid=container_id) # todo for delete----- print('debug->query1 : {q}'.format(q=query)) # ------------------------ dbi.update_asQuery(query) client = MongoClient('localhost', 27017) db = client['microblog_spider'] assemble_table = db.assemble_factory res = assemble_table.find({'container_id': container_id}, { 'current_id': 1, 'total_num': 1 }) id_list = [x['current_id'] for x in res] num = int([ x['total_num'] for x in assemble_table.find({ 'container_id': container_id }).limit(1) ][0]) ## todo for delete----- print('debug->id_list_len: {len}'.format(len=id_list.__len__())) print('debug->num: {n}'.format(n=num)) # ------------------------ # 检查是否所有包裹已经到齐 check_state = True if id_list.__len__() < num: print( 'server->HistoryReport:The package is not complete, retry to catch data' ) check_state = False if check_state: # 如果所有子包已经收集完毕,则将数据放入正式数据库mongodb # 将装配车间中的相关数据删除 # 并且在Mysql中更新update_time和latest_blog,抹掉isGettingBlog # 从mysql获取该用户信息 try: query = 'select * from user_info_table where container_id=\'{cid}\'' \ .format(cid=container_id) user_info = dbi.select_asQuery(query)[0] # todo fro debug------------- print('task {cid} :debug->query2: {q}'.format( q=query, cid=container_id)) print('task {cid} debug->user_info:'.format( cid=container_id)) print(user_info) # -------------------------------- col_name = dbi.get_col_name('user_info_table') except Exception as e: print( 'task {cid} :Error:server-HistoryReturn:' 'No such user in MySQL.user_info_table,Reason:'.format( cid=container_id)) print(e) # 将数据从assemble factory中提取出来 try: data_list = assemble_table.find( {'container_id': container_id}, { 'data': 1, 'current_id': 1 }) data_list_ori = [x for x in data_list] data_list = [x['data'] for x in data_list_ori] id_list = [x['current_id'] for x in data_list_ori] data_list_ori = None # todo fro debug------------- print('task {cid} debug->datalist: {len}'.format( len=data_list.__len__(), cid=container_id)) # -------------------------------- except Exception as e: print( 'Error:server-HistoryReturn:' 'Unable to get data from MongoDB, assemble factory,Reason:' ) print(e) # 长度大于预期,说明有重复信息,需要去重 if id_list.__len__() > num: unique_data_list = [] check_dict = {} for i in range(id_list.__len__()): try: # 这里使用字典去重,(算是hash吧) check_dict[str(id_list[i])] continue except: check_dict[str(id_list[i])] = True unique_data_list.append(data_list[i]) # print('data_list.len :{len}'.format(len=data_list.__len__())) # print('id_list.len :{len}'.format(len=id_list.__len__())) # print(i) data_list = unique_data_list # 将碎片拼接 try: data_final = [] for i in data_list: data_final = data_final + i # todo fro debug------------- print('task {cid} debug->数据拼接完毕,len {len}'.format( len=data_final.__len__(), cid=container_id)) # -------------------------------- except Exception as e: print( 'Error:server-HistoryReport:' 'Unable to contact the pieces of information,Reason:') print(e) # 将本次信息录入accuracy_table 用以进一步分析 blog_len = data_final.__len__() wanted_blog_len = user_info[col_name.index('blog_num')] blog_accuracy = blog_len / wanted_blog_len time_stick = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) query = 'insert into accuracy_table values ({acc},\'{t_s}\',{num}) ;' \ .format(acc=blog_accuracy, t_s=time_stick, num=wanted_blog_len) dbi.insert_asQuery(query) # 将数据录入Mongodb 更改Mysql,删除assemble中相关内容 try: if not user_info[col_name.index('update_time')]: # 将数据存入 Mongodb 的formal collection save_data_seperately(data_final) print( 'task {cid} Success: Data has saved in Mongodb, size is {size}' .format(size=sys.getsizeof(data_final), cid=container_id)) # # 将关键信息录入Mydql query = 'update user_info_table set ' \ 'update_time=\'{up_time}\',' \ 'latest_blog=\'{latest_blog}\',' \ 'isGettingBlog=null ' \ 'where container_id=\'{cid}\';'\ .format(up_time=time_stick,latest_blog=latest_time,cid=container_id) # query='update user_info_table set ' \ # 'update_time=\'{up_time}\',' \ # 'latest_blog=\'{latest_blog}\'' \ # 'where container_id=\'{cid}\';' \ # .format(up_time=time_stick,latest_blog=latest_time,cid=container_id) #TODO 这里为了方便统计,去掉了抹除isGetting这一项,但是正式运行的时候是要加上的 dbi.update_asQuery(query) print( 'task {cid} Success: insert user into MongoDB, the num of data is {len}' .format(len=blog_len, cid=container_id)) else: query='update user_info_table set isGettingBlog=null where container_id=\'{cid}\'' \ .format(cid=container_id) dbi.update_asQuery(query) except Exception as e: print('task {cid} Error:server->HistoryReport:' 'Reason:'.format(cid=container_id)) print(e) else: # 如果所有子包不全,则抹掉isGettingBlog,将装配车间中数据删除 print( 'task {cid} :Error: the package is not complete ,{a} of {b}' .format(a=id_list.__len__(), b=num, cid=container_id)) query='update user_info_table set isGettingBlog=null where container_id=\'{cid}\'' \ .format(cid=container_id) dbi.update_asQuery(query) # 将数据从assemble factory 去掉 assemble_table.remove({'container_id': container_id}) print( 'task {cid} Success: Data has been removed from assemble factory' .format(cid=container_id)) # 将cache_history中的相应行删掉,表示已经处理完该事物了 query='delete from cache_history where container_id=\'{cid}\'' \ .format(cid=container_id) dbi.update_asQuery(query) end_time = time.time() deal_time = end_time - start_time print( 'task {cid} :Success : the user {cid} is completed, length is {len}, use {t} seconds' .format(cid=container_id, len=data_final.__len__(), t=deal_time))
def get(self): global proxy uuid = str(self.get_argument('uuid')) task_id = self.task_assign(uuid) if proxy.get_ave_proxy_size( ) < 30: # check the size of current proxy size self.write('no task') self.finish() return if task_id == -1: # checi if this uuid is valid self.write('no task') self.finish() return if task_id == 1: # get the social web of certain user dbi = MySQL_Interface() query = 'select * from ready_to_get where is_fetching is null order by fans_num desc limit 1;' res = dbi.select_asQuery(query) if res.__len__() == 0: self.write('no task') self.finish() return res = res[0] col_info = dbi.get_col_name('ready_to_get') uid = res[col_info.index('uid')] self.write('{uid},connect'.format(uid=uid)) self.finish() time_stick = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) query="update ready_to_get set is_fetching=\'{t_time}\' where uid={uid} ;"\ .format(t_time=time_stick,uid=uid) dbi.update_asQuery(query) if task_id == 2: # get the history microblog of a certain user dbi = MySQL_Interface() query='select container_id,blog_num from user_info_table ' \ 'where (isGettingBlog is null and update_time is null and blog_num<{valve} and blog_num>100)' \ 'order by fans_num desc limit 1 ;'.format(valve=config.HISTORY_TASK_VALVE) # query='select container_id,blog_num from user_info_table ' \ # 'order by rand() limit 1 ;' res = dbi.select_asQuery(query) if res.__len__() == 0: self.write('no task') self.finish() return [container_id, blog_num] = res[0] self.write('{c_id};{blog},history'.format(c_id=container_id, blog=blog_num)) self.finish() time_stick = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) query="update user_info_table set isGettingBlog=\'{t_time}\' where container_id={cid} ;"\ .format(t_time=time_stick,cid=container_id) dbi.update_asQuery(query) if task_id == 3: # get the history microblog of a certain user dbi = MySQL_Interface() query='select container_id,blog_num from user_info_table ' \ 'where (isGettingBlog is null and update_time is null and blog_num>={valve} and blog_num>100)' \ 'order by fans_num desc limit 1 ;'.format(valve=config.HISTORY_TASK_VALVE) # query='select container_id,blog_num from user_info_table ' \ # 'order by rand() limit 1 ;' [container_id, blog_num] = dbi.select_asQuery(query)[0] self.write('{c_id};{blog},history'.format(c_id=container_id, blog=blog_num)) self.finish() time_stick = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) query="update user_info_table set isGettingBlog=\'{t_time}\' where container_id={cid} ;" \ .format(t_time=time_stick,cid=container_id) dbi.update_asQuery(query) if task_id == 4 or task_id == 5 or task_id == 100: # this part is in test dbi = MySQL_Interface() current_time_stick = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) target_time_stick = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 60 * 60 * 24 * 1)) #提早5天 if task_id == 4: batch_size = 100 elif task_id == 5: batch_size = 200 else: batch_size = 10 query='select container_id,update_time,latest_blog from user_info_table ' \ 'where update_time<\'{target_time}\' and isGettingBlog is null and blog_num>10 order by fans_num desc limit {batch}' \ .format(target_time=target_time_stick,batch=batch_size) print(query) res = dbi.select_asQuery(query) # 将从mysql中取得的用户列表加上必要的变量以后发送给客户端 res = [[ line[0], int(time.mktime(line[1].timetuple())), int(time.mktime(line[2].timetuple())) ] for line in res] res_cp = res if res_cp.__len__() == 0: # if no task ,then return "no task" print('*** warning: no avaliable update mission ***') self.write('no task') self.finish() return # print('debug from task handler') # pprint(res_cp) res = [ line[0] + '-' + str(line[1]) + '-' + str(line[2]) for line in res ] inn = '' for item in res: inn += item + ';' inn = inn[0:-1] # uid-stamp;uid-timestamp;...;,update (the formation of order) mission_id = random_str(15) commend = '{list};{task_id},update'.format(list=inn, task_id=mission_id) # 传送给客户端的指令格式: ContainerId-UpdateTime-LatestBlog;...;...;...,update self.write(commend) self.finish() # 将用户列表,任务id,以及任务开始时间存入mongodb u_list = [ dict(container_id=x[0], update_time=x[1], latest_blog=x[2]) for x in res_cp ] data_toMongo = dict(mission_id=mission_id, user_list=u_list, mission_start=int(time.time())) client = MongoClient('localhost', 27017) db = client['microblog_spider'] collec = db.update_mission collec.insert(data_toMongo) # 将相关内容从mysql中设置isGettingBlog user_list_str = '' for line in res_cp: user_list_str += '\'{cid}\','.format(cid=line[0]) user_list_str = user_list_str[:-1] time_stick = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) query='update user_info_table set isGettingBlog=\'{time}\' where container_id in ({ulist})'\ .format(time=time_stick,ulist=user_list_str) dbi.update_asQuery(query)
def run(self): while True: start_time = time.time() dbi=MySQL_Interface() col_info=dbi.get_col_name('cache_history') query='select * from cache_history where is_dealing is null order by checkin_timestamp limit 1' mysql_res=dbi.select_asQuery(query) if mysql_res.__len__()==0: # cache_history表为空时,睡眠1秒,跳过此次循环 time.sleep(1) continue mysql_res=mysql_res[0] # todo for delete----- print('debug->start to deal with a new task') print('debug->mysql_res: ') print(mysql_res) #------------------------ container_id=mysql_res[col_info.index('container_id')] print('debug->container_id: {cid}'.format(cid=container_id)) latest_time=mysql_res[col_info.index('latest_time')] latest_timestamp=mysql_res[col_info.index('latest_timestamp')] time_stick=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) query = 'update cache_history set is_dealing=\'{time}\' where container_id={cid}'.format(time=time_stick, cid = container_id) # todo for delete----- print('debug->query1 : {q}'.format(q=query)) # ------------------------ dbi.update_asQuery(query) client = MongoClient('localhost', 27017) db = client['microblog_spider'] assemble_table = db.assemble_factory res = assemble_table.find({'container_id': container_id}, {'current_id': 1, 'total_num': 1}) id_list = [x['current_id'] for x in res] num = int([x['total_num'] for x in assemble_table.find({'container_id': container_id}).limit(1)][0]) ## todo for delete----- print('debug->id_list_len: {len}'.format(len=id_list.__len__())) print('debug->num: {n}'.format(n=num)) # ------------------------ # 检查是否所有包裹已经到齐 check_state = True if id_list.__len__() < num: print('server->HistoryReport:The package is not complete, retry to catch data') check_state = False if check_state: # 如果所有子包已经收集完毕,则将数据放入正式数据库mongodb # 将装配车间中的相关数据删除 # 并且在Mysql中更新update_time和latest_blog,抹掉isGettingBlog # 从mysql获取该用户信息 try: query = 'select * from user_info_table where container_id=\'{cid}\'' \ .format(cid=container_id) user_info = dbi.select_asQuery(query)[0] # todo fro debug------------- print('task {cid} :debug->query2: {q}'.format(q=query,cid=container_id)) print('task {cid} debug->user_info:'.format(cid = container_id)) print(user_info) # -------------------------------- col_name = dbi.get_col_name('user_info_table') except Exception as e: print('task {cid} :Error:server-HistoryReturn:' 'No such user in MySQL.user_info_table,Reason:'.format(cid = container_id)) print(e) # 将数据从assemble factory中提取出来 try: data_list = assemble_table.find({'container_id':container_id}, {'data': 1 , 'current_id': 1}) data_list_ori = [x for x in data_list] data_list = [x['data'] for x in data_list_ori] id_list = [x['current_id'] for x in data_list_ori] data_list_ori = None # todo fro debug------------- print('task {cid} debug->datalist: {len}'.format(len = data_list.__len__(),cid=container_id)) # -------------------------------- except Exception as e: print('Error:server-HistoryReturn:' 'Unable to get data from MongoDB, assemble factory,Reason:') print(e) # 长度大于预期,说明有重复信息,需要去重 if id_list.__len__() > num : unique_data_list = [] check_dict = {} for i in range(id_list.__len__()) : try: # 这里使用字典去重,(算是hash吧) check_dict[str(id_list[i])] continue except: check_dict[str(id_list[i])] = True unique_data_list.append(data_list[i]) # print('data_list.len :{len}'.format(len=data_list.__len__())) # print('id_list.len :{len}'.format(len=id_list.__len__())) # print(i) data_list = unique_data_list # 将碎片拼接 try: data_final = [] for i in data_list: data_final = data_final+i # todo fro debug------------- print('task {cid} debug->数据拼接完毕,len {len}'.format(len=data_final.__len__(),cid=container_id)) # -------------------------------- except Exception as e: print('Error:server-HistoryReport:' 'Unable to contact the pieces of information,Reason:') print(e) # 将本次信息录入accuracy_table 用以进一步分析 blog_len = data_final.__len__() wanted_blog_len = user_info[col_name.index('blog_num')] blog_accuracy = blog_len/wanted_blog_len time_stick = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) query = 'insert into accuracy_table values ({acc},\'{t_s}\',{num}) ;' \ .format(acc=blog_accuracy, t_s=time_stick, num=wanted_blog_len) dbi.insert_asQuery(query) # 将数据录入Mongodb 更改Mysql,删除assemble中相关内容 try: if not user_info[col_name.index('update_time')]: # 将数据存入 Mongodb 的formal collection save_data_seperately(data_final) print('task {cid} Success: Data has saved in Mongodb, size is {size}' .format(size=sys.getsizeof(data_final),cid=container_id)) # # 将关键信息录入Mydql query = 'update user_info_table set ' \ 'update_time=\'{up_time}\',' \ 'latest_blog=\'{latest_blog}\',' \ 'isGettingBlog=null ' \ 'where container_id=\'{cid}\';'\ .format(up_time=time_stick,latest_blog=latest_time,cid=container_id) # query='update user_info_table set ' \ # 'update_time=\'{up_time}\',' \ # 'latest_blog=\'{latest_blog}\'' \ # 'where container_id=\'{cid}\';' \ # .format(up_time=time_stick,latest_blog=latest_time,cid=container_id) #TODO 这里为了方便统计,去掉了抹除isGetting这一项,但是正式运行的时候是要加上的 dbi.update_asQuery(query) print('task {cid} Success: insert user into MongoDB, the num of data is {len}' .format(len=blog_len,cid=container_id)) else: query='update user_info_table set isGettingBlog=null where container_id=\'{cid}\'' \ .format(cid=container_id) dbi.update_asQuery(query) except Exception as e: print('task {cid} Error:server->HistoryReport:' 'Reason:'.format(cid=container_id)) print(e) else: # 如果所有子包不全,则抹掉isGettingBlog,将装配车间中数据删除 print('task {cid} :Error: the package is not complete ,{a} of {b}' .format(a=id_list.__len__(),b=num,cid=container_id)) query='update user_info_table set isGettingBlog=null where container_id=\'{cid}\'' \ .format(cid=container_id) dbi.update_asQuery(query) # 将数据从assemble factory 去掉 assemble_table.remove({'container_id':container_id}) print('task {cid} Success: Data has been removed from assemble factory' .format(cid=container_id)) # 将cache_history中的相应行删掉,表示已经处理完该事物了 query='delete from cache_history where container_id=\'{cid}\'' \ .format(cid=container_id) dbi.update_asQuery(query) end_time = time.time() deal_time = end_time - start_time print('task {cid} :Success : the user {cid} is completed, length is {len}, use {t} seconds' .format(cid = container_id, len = data_final.__len__(), t = deal_time))
self.bit_size=1<<15 self.seeds=[5,7,11,13,31,37,61] self.r=redis.StrictRedis(host='127.0.0.1',port=6379,db=0) self.hashFunc=[] for i in range(self.seeds.__len__()): self.hashFunc.append(SimpleHash(self.bit_size,self.seeds[i])) def isContains(self,str_input,name): if str_input==None: return False if str_input.__len__()==0: return False ret=True for f in self.hashFunc: loc=f.hash(str_input) ret=ret & self.r.getbit(name,loc) return ret def insert(self,str_input,name): for f in self.hashFunc: loc=f.hash(str_input) self.r.setbit(name,loc,1) dbi=MySQL_Interface(dbname='microblog_spider') r=redis.StrictRedis(host='127.0.0.1',port=6379,db=0) query='select uid from user_info_table ;' uid=dbi.select_asQuery(query) uid=[x[0] for x in uid] bf=BloomFilter() for id in uid: bf.insert(id,'user_info_table')
self.seeds = [5, 7, 11, 13, 31, 37, 61] self.r = redis.StrictRedis(host='127.0.0.1', port=6379, db=0) self.hashFunc = [] for i in range(self.seeds.__len__()): self.hashFunc.append(SimpleHash(self.bit_size, self.seeds[i])) def isContains(self, str_input, name): if str_input == None: return False if str_input.__len__() == 0: return False ret = True for f in self.hashFunc: loc = f.hash(str_input) ret = ret & self.r.getbit(name, loc) return ret def insert(self, str_input, name): for f in self.hashFunc: loc = f.hash(str_input) self.r.setbit(name, loc, 1) dbi = MySQL_Interface(dbname='microblog_spider') r = redis.StrictRedis(host='127.0.0.1', port=6379, db=0) query = 'select uid from user_info_table ;' uid = dbi.select_asQuery(query) uid = [x[0] for x in uid] bf = BloomFilter() for id in uid: bf.insert(id, 'user_info_table')