task_Q.put(Q)
self.f_cnt += 1
def commit_link(self, Qlist, p_size):
pipe = self.r.pipeline()
for i in xrange(p_size):
new_Qs = Qlist.get()
for Q in new_Qs:
pipe.rpush(self.commit_k, Q)
pipe.execute()
self.c_cnt += 1
if __name__ == '__main__':
bar = prog_bar.prog_bar(100)
slave_NO, curren_f, delay = get_arg()
filename = "../yahoo_" + str(slave_NO) + str(curren_f) + ".txt"
yh_of = open(filename, 'a+')
r = redis.StrictRedis(host=master, port=r_port, db=0)
slaver = worker(r, sla_cnt)
task_Q = Queue.Queue()
fresh_Q = Queue.Queue()
pool = threadpool.ThreadPool(thread_cnt)
slaver.init_work()
while slaver.unfinished:
if __name__ == '__main__': pool = threadpool.ThreadPool(thread_cnt) start_t=time.time() delay,urlcapacity=get_arg() Ques_f,relay=init_filter(urlcapacity) url_Q=url_Queue(Ques_f) r = redis.StrictRedis(host='spider01', port=6369, db=0) master=scheduler(r,sla_cnt,size=salve_job) bar=prog_bar.prog_bar(total_p) questions,sids=get_question(start_url) url_Q.pour(questions) ques_works=start_working(start_p,end_p,relay) while len(url_Q.filter)<urlcapacity: t=time.time()-start_t wait_q=url_Q.Q.qsize() bar.reflash(t,url_Q.length(),master.in_q,wait_q,master.out_q,urlcapacity)
return self.r.llen(key)
if __name__ == "__main__":
pool = threadpool.ThreadPool(thread_cnt)
start_t = time.time()
delay, urlcapacity = get_arg()
Ques_f, relay = init_filter(urlcapacity)
url_Q = url_Queue(Ques_f)
r = redis.StrictRedis(host="spider01", port=6369, db=0)
master = scheduler(r, sla_cnt, size=salve_job)
bar = prog_bar.prog_bar(total_p)
questions, sids = get_question(start_url)
url_Q.pour(questions)
ques_works = start_working(start_p, end_p, relay)
while len(url_Q.filter) < urlcapacity:
t = time.time() - start_t
wait_q = url_Q.Q.qsize()
bar.reflash(t, url_Q.length(), master.in_q, wait_q, master.out_q, urlcapacity)
master.dist_all("task_url", url_Q)
if wait_q < in_time:
master.retirve("fresh_url", url_Q)
self.f_cnt+=1 def commit_link(self,Qlist,p_size): pipe=self.r.pipeline() for i in xrange(p_size): new_Qs=Qlist.get() for Q in new_Qs: pipe.rpush(self.commit_k,Q) pipe.execute() self.c_cnt+=1 if __name__ == '__main__': bar=prog_bar.prog_bar(100) slave_NO,curren_f,delay=get_arg() filename="../yahoo_"+str(slave_NO)+str(curren_f)+".txt" yh_of=open(filename,'a+') r = redis.StrictRedis(host=master, port=r_port,db=0) slaver=worker(r,sla_cnt) task_Q=Queue.Queue() fresh_Q=Queue.Queue() pool = threadpool.ThreadPool(thread_cnt) slaver.init_work() while slaver.unfinished:
