def test_addtask(self):
sp = Spider(SPIDER)
sp.frontier.clean('todo')
sp.addtask('http://www.google.com')
self.assertEqual(1, len(sp.frontier))
sp.frontier.clean('todo')
del sp
def test_run(self):
sp = Spider(SPIDER)
sp.frontier.clean('todo', 'visited')
url = ['http://www.baidu.com', 'http://www.zhihu.com']
sp.addtask(url)
sp.run()
self.assertEqual(0, len(sp.frontier))
self.assertEqual(2, sp.frontier.redis.llen(sp.frontier.visited))
sp.frontier.clean('todo', 'visited')
self.assertTrue(sp.handlers[0]._exists(url[0]))
self.assertTrue(sp.handlers[0]._exists(url[1]))
del sp
