class TestProcessThreads(unittest.TestCase): def setUp(self): mockweb = MockWebClient(HTMLFactory.tablamensajes_html()) self.threadfactory = AsuntoFactory(mockweb) self.mongo = MockMongo() self.processthread = ProcessThreads(self.mongo, MockMsgPageFactory()) self.navhtml = HTMLFactory.tablamensajes_html() + HTMLFactory.navigation_url() def test_getlistofthreads(self): self.assertEquals("", self.threadfactory.nextUrl()) self.processthread.storeThreads(self.threadfactory) self.assertEquals(2, self.mongo.treadssaved) def test_createThreadStruct(self): self.assertEquals("", self.threadfactory.nextUrl()) thread = self.threadfactory.create(HTMLFactory.asunto()) msgs = list() struct = self.processthread._createThreadStruct(thread, msgs) self.assertEquals("LaBSK", struct['source']) self.assertEquals(thread['title'], struct['title']) self.assertIsInstance(struct['msgs'], list) def test_createThreadStruct_with_msgs(self): self.processthread.storeThreads(self.threadfactory) self.assertEquals(2, len(self.mongo.listofthreads)) thread = self.mongo.listofthreads[0] self.assertEqual(2, len(thread['msgs'])) def test_default_pagelimit_is_1(self): mockweb = MockWebClient(self.navhtml) self.processthread.storeThreads(AsuntoFactory(mockweb)) self.assertEquals(2, len(self.mongo.listofthreads)) def test_two_loops(self): mockweb = MockWebClient(self.navhtml) self.processthread.pagelimit = 2 self.processthread.storeThreads(AsuntoFactory(mockweb)) self.assertEquals(4, len(self.mongo.listofthreads)) def test_generate_new_factory(self): oldpage = AsuntoFactory( MockWebClient(self.navhtml)) page = self.processthread._nextPage(oldpage) self.assertEquals(u"http://labsk.net/index.php?board=18.20", page.webclient.url)
""" Old thread seems to be the same one than the new thread """ self.thread -= 1 print "Skipping ", new.title, ", ", new.date, " | ", new.link def __str__(self): return ". Threads: " + str(self.thread) \ db = MongoDB() db.query("labsk_merge") db.insert("labsk_" + str(datetime.now())) starttime = datetime.now() listener = StdListener() threads = ProcessThreads(db, MsgPageFactory()) threads.setListener(listener) threads.setPageLimit(1) threads.setMsgPageLimit(250) # Nunca bajes este valor o perderas mensajes, al menos mantenlo igual threads.scrapListOfURL(labsk_urls) delta = datetime.now() - starttime print "----------------------------------------------" print "Total time: ", delta print "Page limit ", threads.pagelimit, " Msg page limit ", threads.msgpagelimit print str(listener) mr = db.merge('link') print str(mr)
def setUp(self): mockweb = MockWebClient(HTMLFactory.tablamensajes_html()) self.threadfactory = AsuntoFactory(mockweb) self.mongo = MockMongo() self.processthread = ProcessThreads(self.mongo, MockMsgPageFactory()) self.navhtml = HTMLFactory.tablamensajes_html() + HTMLFactory.navigation_url()
class TestProcessThreads(unittest.TestCase): def setUp(self): mockweb = MockWebClient(HTMLFactory.tablamensajes_html()) self.threadfactory = AsuntoFactory(mockweb) self.mongo = MockMongo() self.processthread = ProcessThreads(self.mongo, MockMsgPageFactory()) self.navhtml = HTMLFactory.tablamensajes_html() + HTMLFactory.navigation_url() def test_getlistofthreads(self): self.assertEquals("", self.threadfactory.nextUrl()) self.processthread.storeThreads(self.threadfactory) self.assertEquals(2, self.mongo.treadssaved) def test_createThreadStruct_with_msgs(self): self.processthread.storeThreads(self.threadfactory) self.assertEquals(2, len(self.mongo.listofthreads)) thread = self.mongo.listofthreads[0] self.assertEqual(2, len(thread['msgs'])) def test_createThreadStruct_stores_locaton(self): urlobject = mock() when(urlobject).description().thenReturn("X") self.threadfactory.urlobject = urlobject self.processthread.storeThreads(self.threadfactory) self.assertEquals(2, len(self.mongo.listofthreads)) thread = self.mongo.listofthreads[0] self.assertEqual("X", thread['location']) def test_default_pagelimit_is_1(self): mockweb = MockWebClient(self.navhtml) self.processthread.storeThreads(AsuntoFactory(mockweb)) self.assertEquals(2, len(self.mongo.listofthreads)) def test_two_loops(self): mockweb = MockWebClient(self.navhtml) self.processthread.pagelimit = 2 self.processthread.storeThreads(AsuntoFactory(mockweb)) self.assertEquals(4, len(self.mongo.listofthreads)) def test_generate_new_factory(self): oldpage = AsuntoFactory( MockWebClient(self.navhtml)) page = self.processthread._nextPage(oldpage) self.assertEquals(u"http://labsk.net/index.php?board=18.20", page.webclient.url)