def crawl(self):
main_thread = threading.currentThread()
domaines=self.db.new_domaines.distinct('domaine')
threadpool=[]
rec=Record(self.db_value)
rec.start()
i=0
for domaine in domaines:
i=i+1
cw=CrawlerThread(domaine,self.db)
cw.start()
if i % 30==0:
for t in threading.enumerate():
if t is not main_thread:
t.join(2)
stop=True
while(stop):
for t in threadpool:
if not t.IsActive():
threadpool.remove(t)
if len(threadpool)==0:
stop=False
