task_Q.put(Q) self.f_cnt += 1 def commit_link(self, Qlist, p_size): pipe = self.r.pipeline() for i in xrange(p_size): new_Qs = Qlist.get() for Q in new_Qs: pipe.rpush(self.commit_k, Q) pipe.execute() self.c_cnt += 1 if __name__ == '__main__': bar = prog_bar.prog_bar(100) slave_NO, curren_f, delay = get_arg() filename = "../yahoo_" + str(slave_NO) + str(curren_f) + ".txt" yh_of = open(filename, 'a+') r = redis.StrictRedis(host=master, port=r_port, db=0) slaver = worker(r, sla_cnt) task_Q = Queue.Queue() fresh_Q = Queue.Queue() pool = threadpool.ThreadPool(thread_cnt) slaver.init_work() while slaver.unfinished:
if __name__ == '__main__': pool = threadpool.ThreadPool(thread_cnt) start_t=time.time() delay,urlcapacity=get_arg() Ques_f,relay=init_filter(urlcapacity) url_Q=url_Queue(Ques_f) r = redis.StrictRedis(host='spider01', port=6369, db=0) master=scheduler(r,sla_cnt,size=salve_job) bar=prog_bar.prog_bar(total_p) questions,sids=get_question(start_url) url_Q.pour(questions) ques_works=start_working(start_p,end_p,relay) while len(url_Q.filter)<urlcapacity: t=time.time()-start_t wait_q=url_Q.Q.qsize() bar.reflash(t,url_Q.length(),master.in_q,wait_q,master.out_q,urlcapacity)
return self.r.llen(key) if __name__ == "__main__": pool = threadpool.ThreadPool(thread_cnt) start_t = time.time() delay, urlcapacity = get_arg() Ques_f, relay = init_filter(urlcapacity) url_Q = url_Queue(Ques_f) r = redis.StrictRedis(host="spider01", port=6369, db=0) master = scheduler(r, sla_cnt, size=salve_job) bar = prog_bar.prog_bar(total_p) questions, sids = get_question(start_url) url_Q.pour(questions) ques_works = start_working(start_p, end_p, relay) while len(url_Q.filter) < urlcapacity: t = time.time() - start_t wait_q = url_Q.Q.qsize() bar.reflash(t, url_Q.length(), master.in_q, wait_q, master.out_q, urlcapacity) master.dist_all("task_url", url_Q) if wait_q < in_time: master.retirve("fresh_url", url_Q)
self.f_cnt+=1 def commit_link(self,Qlist,p_size): pipe=self.r.pipeline() for i in xrange(p_size): new_Qs=Qlist.get() for Q in new_Qs: pipe.rpush(self.commit_k,Q) pipe.execute() self.c_cnt+=1 if __name__ == '__main__': bar=prog_bar.prog_bar(100) slave_NO,curren_f,delay=get_arg() filename="../yahoo_"+str(slave_NO)+str(curren_f)+".txt" yh_of=open(filename,'a+') r = redis.StrictRedis(host=master, port=r_port,db=0) slaver=worker(r,sla_cnt) task_Q=Queue.Queue() fresh_Q=Queue.Queue() pool = threadpool.ThreadPool(thread_cnt) slaver.init_work() while slaver.unfinished: