def __str__(self):
return ". Threads: " + str(self.thread) \
db = MongoDB()
db.query("labsk_merge")
db.insert("labsk_" + str(datetime.now()))
starttime = datetime.now()
listener = StdListener()
threads = ProcessThreads(db, MsgPageFactory())
threads.setListener(listener)
threads.setPageLimit(1)
threads.setMsgPageLimit(250) # Nunca bajes este valor o perderas mensajes, al menos mantenlo igual
threads.scrapListOfURL(labsk_urls)
delta = datetime.now() - starttime
print "----------------------------------------------"
print "Total time: ", delta
print "Page limit ", threads.pagelimit, " Msg page limit ", threads.msgpagelimit
print str(listener)
mr = db.merge('link')
print str(mr)
print "Scrapping planeta ludico"
blogs = PlanetaLudicoScrap(db.blogs_collection())
