def main(): url_task_queue = Queue.Queue() thread_searcher_pool = ThreadPool(500, url_task_queue) thread_searcher_pool.start() topic_searcher_urls_pool = ThreadPool(200) topic_searcher_urls_pool.start() topic_threader = TopicSearcherThreader( thread_searcher_pool, topic_searcher_urls_pool, build_url_job, HenHenLuTopicUrlExer, BASE_URL, START_NUM, END_NUM, PIC_TYPE, ) topic_threader.start() while True: print '#######################################' print 'URL has task count: %s' \ % str(thread_searcher_pool.current_task_count()) print '#######################################' print '#######################################' print 'TOPIC has task count: %s' \ % str(topic_searcher_urls_pool.current_task_count()) print '#######################################' time.sleep(5) topic_threader.join() topic_searcher_urls_pool.wait_done() thread_searcher_pool.wait_done()
def main1(): thread_pool = ThreadPool(20) thread_pool.start() session = get_session() topic_query = \ session.query(VideoTopic).filter(VideoTopic.video_type == 1) for topic in topic_query: thread_pool.add_task(job, topic.henhen_id) session.close() thread_pool.wait_done()
def main(): pool = ThreadPool(20) pool.start() session = get_session() topic_query = session.query(PicTopic).filter(PicTopic.pic_type == 'dongmantupian').order_by(PicTopic.id.desc()) for pic_topic in topic_query: pool.add_task(dump_job, pic_topic) session.close() pool.wait_done()
def main(): thread_pool = ThreadPool(50) thread_pool.start() video_type = '7' base_url = 'http://www.toutoulu.com/vodlist/%s_%s.html' # init task for page_num in range(1, page_info[video_type] + 1): url = base_url % (video_type, page_num) print 'add task %s' % url thread_pool.add_task(thread_pool_job, url, video_type) thread_pool.wait_done()