def wait_tasks_done(pool):
for t in pool:
if not t.isAlive():
pool.remove(t)
else:
t.join()
if __name__ == "__main__":
tick_start = time.time()
pool = []
while True:
# Get a topic to grab its content
topic = mysql_mgr.dequeue_topic()
if topic is None:
wait_tasks_done()
exit(1)
task = Thread(target=post_crawl_task, args=(topic, ))
task.start()
pool.append(task)
if len(pool) == max_threads:
wait_tasks_done(pool)
dur_task_run = time.time() - tick_start
if dur_task_run < interval:
time.sleep(interval - dur_task_run)
tick_start = time.time()
