class NovelCrawler:
def __init__(self):
self.crawler = CrawlerService()
#service
db = mongo.MongoDB(config=mongo.conf)
self.book_server = BookServer(db)
self.novel_server = NovelServer(db)
def crawl(self, input):
for url in imap(string.strip, input):
novel = self.crawler.crawl(url)
#save novel
succ = self.novel_server.add_or_update(novel)
if not succ:
logger.info("save novel %s failed" % novel['nid'])
return False
else:
logger.info("save novel %s success" % novel['nid'])
#convert novel to book
logger.debug("Update Book tables")
#save book
succ = self.book_server.add_or_update(novel)
if not succ:
logger.info("save book %s failed" % novel['bid'])
return False
else:
logger.info("save book %s success" % novel['bid'])
return True
def __init__(self):
self.crawler = CrawlerService()
#service
db = mongo.MongoDB(config=mongo.conf)
self.book_server = BookServer(db)
self.novel_server = NovelServer(db)