def forum_config():
#加载论坛列表
#forumList=[{'forumId':1,'forumName':'体育'},{'forumId':2,'forumName':'美食'}]
ms = MongoStore()
ms.open()
forumList = ms.find('forums',{})
return render_template("forum_config.html",forumList=forumList)
def add_forum():
forum = ForumInfo()
forum.forumId=request.form['forumId']
forum.forumName=request.form['forumName']
forum._id=forum.forumId
print(forum.__dict__)
ms = MongoStore()
ms.open()
ms.saveForum(forum)
return "OK"
def get_forum():
forumId=request.args.get('forumId', 0)
print(forumId)
ms = MongoStore()
ms.open()
forumList = ms.find('forums',{'forumId':forumId})
if forumList:
#输出json
return jsonify(forumList[0])
return jsonify(ForumInfo().__dict__)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
' 主程序 '
#from scanner.xitekInfo import ThreadInfo,PostInfo,PageInfo,ForumInfo
from scanner.scanner_thread import ThreadScanner
from scanner.scanner_forum import ForumScanner
from mongo_store import MongoStore
#准备数据库
ms = MongoStore()
ms.open()
#执行抓取论坛主题
class ProcessForum():
def process(self,forumId,batchId):
#遍历一个forum
pageNum =1
scanner = ForumScanner(forumId)
pageData= scanner.fetchPage(pageNum)
ret = scanner.parsePage(pageData)
threadList = ret[0]
pageInfo = ret[1]
print("****第%s页****"%pageNum)
for p in threadList:
#print (p.threadId+","+p.title+","+p.uid+","+p.uname+","+p.replyNum+"," + p.readNum)
p.batchId=batchId
ms.saveThread(p)
