def test_execute_when_pn(self): """ 测试execute, 当pn 非0 """ novel = self.db_add_novel(name=u'择天记', rule=u'^【择天记】.+第.+章.+$') parser = NovelParser(novel) count, chapter_list = parser.execute(100) self.assertTrue(count) self.assertTrue(chapter_list)
def add_chapters(self, novel, task=None): """ 添加章节列表 :task: 如果不为None,则为使用异步队列 """ if novel.last_sync_time and (novel.last_sync_time > datetime.now() - timedelta(minutes=10)): return False, u'【错误】更新过于频繁' novel.last_sync_time = datetime.now() self.db.add(novel) self.db.commit() parser = NovelParser(novel) pn = 0 count = 1000 while True: try: c, chapter_list = parser.execute(pn) count = c or count for chapter in chapter_list: if self.db.query(Chapter).filter(Chapter.pageid==chapter.pageid).first(): continue chapter.novel_id = novel.id self.db.add(chapter) self.db.commit() except: self.logger.error('Add chapters error|%s|%s|' %(novel.id, pn), exc_info=1) self.db.rollback() pn += 50 if task: task.update_state(state='PROGRESS', meta={'percent': min(pn/float(count), 1.0) * 100}) if pn > count: break return True, ''