def main():
url_task_queue = Queue.Queue()
thread_searcher_pool = ThreadPool(500, url_task_queue)
thread_searcher_pool.start()
topic_searcher_urls_pool = ThreadPool(200)
topic_searcher_urls_pool.start()
topic_threader = TopicSearcherThreader(
thread_searcher_pool,
topic_searcher_urls_pool,
build_url_job,
HenHenLuTopicUrlExer,
BASE_URL,
START_NUM,
END_NUM,
PIC_TYPE,
)
topic_threader.start()
while True:
print '#######################################'
print 'URL has task count: %s' \
% str(thread_searcher_pool.current_task_count())
print '#######################################'
print '#######################################'
print 'TOPIC has task count: %s' \
% str(topic_searcher_urls_pool.current_task_count())
print '#######################################'
time.sleep(5)
topic_threader.join()
topic_searcher_urls_pool.wait_done()
thread_searcher_pool.wait_done()
def main1():
thread_pool = ThreadPool(20)
thread_pool.start()
session = get_session()
topic_query = \
session.query(VideoTopic).filter(VideoTopic.video_type == 1)
for topic in topic_query:
thread_pool.add_task(job, topic.henhen_id)
session.close()
thread_pool.wait_done()
def main():
pool = ThreadPool(20)
pool.start()
session = get_session()
topic_query = session.query(PicTopic).filter(PicTopic.pic_type
== 'dongmantupian').order_by(PicTopic.id.desc())
for pic_topic in topic_query:
pool.add_task(dump_job, pic_topic)
session.close()
pool.wait_done()
def main():
thread_pool = ThreadPool(50)
thread_pool.start()
video_type = '7'
base_url = 'http://www.toutoulu.com/vodlist/%s_%s.html'
# init task
for page_num in range(1, page_info[video_type] + 1):
url = base_url % (video_type, page_num)
print 'add task %s' % url
thread_pool.add_task(thread_pool_job, url, video_type)
thread_pool.wait_done()
