def gevent_worker(option): while True: url = red.lpop(red_queue) if not url: break create_new_slave(url, option)
count = 0 # choose the running way of using database or not try: option = sys.argv[1] except: option = '' if "mongo" not in option: option = "print_data_out" i = 0 # the start page red.lpush(red_queue, "https://www.zhihu.com/people/gaoming623") url = red.lpop(red_queue) create_new_slave(url, option=option) for i in range(50): url = red.lpop(red_queue) create_new_slave(url, option=option) process_pool = Pool(multiprocessing.cpu_count() * 2) process_pool.map_async(process_worker, option) process_pool.close() process_pool.join() ''' while(True): url_list=[] for i in range(50):
count = 0 # choose the running way of using database or not try: option = sys.argv[1] except: option = '' if "mongo" not in option: option = "print_data_out" i = 0 # the start page red.lpush(red_queue, "https://www.zhihu.com/people/gaoming623") url = red.lpop(red_queue) create_new_slave(url, option=option) for i in range(50): url = red.lpop(red_queue) create_new_slave(url, option=option) process_pool = Pool(multiprocessing.cpu_count() * 2) process_pool.map_async(process_worker, option) process_pool.close() process_pool.join() ''' while(True): url_list=[]