def test_addtask(self): sp = Spider(SPIDER) sp.frontier.clean('todo') sp.addtask('http://www.google.com') self.assertEqual(1, len(sp.frontier)) sp.frontier.clean('todo') del sp
def test_run(self): sp = Spider(SPIDER) sp.frontier.clean('todo', 'visited') url = ['http://www.baidu.com', 'http://www.zhihu.com'] sp.addtask(url) sp.run() self.assertEqual(0, len(sp.frontier)) self.assertEqual(2, sp.frontier.redis.llen(sp.frontier.visited)) sp.frontier.clean('todo', 'visited') self.assertTrue(sp.handlers[0]._exists(url[0])) self.assertTrue(sp.handlers[0]._exists(url[1])) del sp