def start(self): if len(self.urllist) == 0: return False self.spider.addurllist(self.urllist) self.spider.setparser(self.parser) self.spider.setindexer(self.indexer) spider.run() return True
self.spider = spider() self.indexer = indexer() self.parser = parser() self.urllist = urllist def start(self): if len(self.urllist) == 0: return False self.spider.addurllist(self.urllist) self.spider.setparser(self.parser) self.spider.setindexer(self.indexer) spider.run() return True def cleanup(self): self.indexer.closedb() if __name__ == "__main__": spider = spider() #spider.addurl('http://localhost:9080/setest/test.php') spider.addurl('http://hq.booksarefun.com/') parserobj = parser() indexobj = indexer() spider.setparser(parserobj) spider.setindexer(indexobj) spider.run() indexobj.closedb() print 'done!'