def getClassify():
try:
db = Db()
result = db.selectAll('SELECT * FROM gysw_classify')
db.close()
return jsonify({'code': '0000', 'message': '请求数据成功', 'data': result})
except:
return jsonify({'code': '9999', 'message': '请求数据失败'})
def addSearch():
try:
open_id = request.json['open_id']
keyword = request.json['keyword']
if not open_id and not keyword:
return jsonify({'code': '9999', 'message': '新增数据失败：参数值不能为空'})
db = Db()
# 取数据
result = db.selectAll(
'select * from gysw_search where keyword = "%s" and open_id = "%s"'
% (keyword, open_id))
if not result or len(result) == 0:
print('新增...')
# 没有，新增
sql = 'insert into gysw_search (`open_id`, `keyword`, `last_update_at`) values (%s, %s, %s)'
localtime = time.localtime(time.time())
db.insertOne(sql, (open_id, keyword, localtime))
else:
print('修改...')
# 修改，次数 +1
current = result[0]
times = current['times'] + 1
db.updateOne(
'update gysw_search set times = %d where open_id = "%s" and keyword = "%s"'
% (times, open_id, keyword))
db.close()
return jsonify({'code': '0000', 'message': '新增数据成功'})
except Exception as e:
print(e)
return jsonify({'code': '9999', 'message': '新增数据失败'})
def getSearchSelf(open_id):
try:
db = Db()
result = db.selectAll(
'select keyword from gysw_search where open_id = "%s" order by last_update_at desc limit 5'
% open_id)
db.close()
return jsonify({'code': '0000', 'message': '查询数据成功', 'data': result})
except Exception as e:
print(e)
return jsonify({'code': '9999', 'message': '查询数据失败'})
def getSearchHot():
try:
db = Db()
result = db.selectAll(
'select keyword, convert(sum(times), int) as times from gysw_search group by keyword order by times desc limit 6'
)
db.close()
print(result)
return jsonify({'code': '0000', 'message': '查询数据成功', 'data': result})
except Exception as e:
print(e)
return jsonify({'code': '9999', 'message': '查询数据失败'})
def getNovel():
keyword = request.args.get('keyword', '')
classify_id = request.args.get('classify_id', '')
open_id = request.args.get('open_id', '')
# 小说表中模糊查询
if classify_id == '' and keyword == '' and open_id == '':
return jsonify({'code': '0000', 'message': '需要传参进行查询', 'data': []})
if classify_id != '':
sql = 'select * from gysw_novel where classify_id = %s' % classify_id
if keyword != '':
sql = 'select * from gysw_novel where book_name like "%%%%%s%%%%" or author_name like "%%%%%s%%%%"' % (
keyword, keyword)
if open_id != '':
sql = 'select * from gysw_shelf where open_id = "%s"' % (open_id, )
try:
db = Db()
result = db.selectAll(sql)
db.close()
if len(result) == 0:
# 根据 url 去笔趣阁查找小说
target_url = 'https://www.biquge5200.cc/modules/article/search.php?searchkey=' + keyword
r = requests.get(target_url)
root = etree.HTML(r.text)
novels = root.xpath('//tr[position()>1]')
result = []
for novel in novels:
author_name = novel.xpath('td[position()=3]/text()')[0]
book_name = novel.xpath('td[position()=1]/a/text()')[0]
url = novel.xpath('td[position()=1]/a/@href')[0]
print(author_name)
print(book_name)
print(url)
result.append({
'author_name': author_name,
'book_name': book_name,
'book_url': url
})
return jsonify({'code': '0000', 'message': '请求数据成功', 'data': result})
except:
return jsonify({'code': '9999', 'message': '请求数据失败'})
def novel():
db = Db()
classifies = db.selectAll('select * from gysw_classify')
for classify in classifies:
target_url = 'https://www.biquge5200.cc/' + classify['path']
try:
r = requests.get(target_url)
root = etree.HTML(r.text)
novel_list = root.xpath('//div[@class="r"]//li')
arr = []
for novel in novel_list:
url = novel.xpath('span[@class="s2"]/a/@href')[0]
book_name = novel.xpath('span[@class="s2"]/a/text()')[0]
author_name = novel.xpath('span[@class="s5"]/text()')[0]
classify_id = classify['id']
arr.append((url, book_name, author_name, classify_id))
print('开始保存数据....')
db.insertMany(
'insert into gysw_novel (`book_url`, `book_name`, `author_name`, `classify_id`) values (%s, %s, %s, %s)',
tuple(arr))
# db.close()
except Exception as e:
print('cuowu chu xian ', e)
print('操作结束')
db.close()
def getShelf():
open_id = request.args.get('open_id', '')
book_name = request.args.get('book_name', '')
author_name = request.args.get('author_name', '')
if not open_id:
return jsonify({'code': '9999', 'message': '查询数据失败：open_id不能为空'})
sql = 'select * from gysw_shelf where `open_id` = "%s"' % open_id
if book_name:
sql += ' and `book_name` = "%s"' % book_name
if author_name:
sql += ' and `author_name` = "%s"' % author_name
try:
db = Db()
result = db.selectAll(sql)
db.close()
return jsonify({'code': '0000', 'message': '查询数据成功', 'data': result})
except:
return jsonify({'code': '9999', 'message': '查询数据失败'})
def reptileNovelByClassify(classify):
db = Db()
target_url = 'https://www.biquge5200.cc/' + classify['path']
try:
r = requests.get(target_url)
root = etree.HTML(r.text)
novel_list = root.xpath('//div[@class="r"]//li')
arr = []
for novel in novel_list:
url = novel.xpath('span[@class="s2"]/a/@href')[0]
book_name = novel.xpath('span[@class="s2"]/a/text()')[0]
author_name = novel.xpath('span[@class="s5"]/text()')[0]
classify_id = classify['id']
arr.append((url, book_name, author_name, classify_id))
print('抓取 %s' % book_name)
print('开始保存数据....')
db.insertMany(
'insert into gysw_novel (`book_url`, `book_name`, `author_name`, `classify_id`) values (%s, %s, %s, %s)',
tuple(arr))
except Exception as e:
print(e)
print('操作结束')
db.close()
def delShelf(id):
try:
db = Db()
db.removeOne('delete from gysw_shelf where id = %s' % id)
db.close()
return jsonify({'code': '0000', 'message': '删除数据成功'})
except:
return jsonify({'code': '9999', 'message': '删除数据失败'})
def editShelf(id):
chapter_url = request.json['chapter_url']
try:
db = Db()
db.updateOne('update gysw_shelf set chapter_url = "%s" where id = %s' %
(chapter_url, id))
db.close()
return jsonify({'code': '0000', 'message': '修改数据成功'})
except:
return jsonify({'code': '9999', 'message': '修改数据失败'})
def addShelf():
try:
open_id = request.json['open_id']
author_name = request.json['author_name']
book_name = request.json['book_name']
chapter_url = request.json['chapter_url']
if not open_id and not author_name and not book_name and not chapter_url:
return jsonify({'code': '9999', 'message': '新增数据失败：参数值不能为空'})
db = Db()
sql = 'insert into gysw_shelf (`open_id`, `author_name`, `book_name`, `chapter_url`) values (%s, %s, %s, %s)'
db.insertOne(sql, (open_id, author_name, book_name, chapter_url))
db.close()
return jsonify({'code': '0000', 'message': '新增数据成功'})
except:
return jsonify({'code': '9999', 'message': '新增数据失败'})
def reptileIndexClassify(self):
print('爬取首页分类数据:开始:(classify/reptileIndexClassify)...')
target_url = 'https://www.biquge5200.com/modules/article/search.php'
try:
r = requests.get(target_url)
root = etree.HTML(r.text)
classifies = root.xpath('//div[@class="nav"]//li[position()>2]')
#
arr1 = []
for classify in classifies:
path = classify.xpath('a/@href')[0].split('/')[-2]
desc = classify.xpath('a/text()')[0]
arr1.append((path, desc))
db = Db()
db.insertMany(
'insert ignore into gysw_classify (`path`, `desc`) values (%s, %s)',
tuple(arr1))
# db.insertOne('insert ignore into gysw_classify(`path`, `desc`) values ("xxx2", "yyy2")')
db.close()
print('爬取首页分类数据:成功:(classify/reptileIndexClassify)...')
except Exception as e:
print('爬取首页分类数据:失败:(classify/reptileIndexClassify)...')
print(e)
from flask_wtf import FlaskForm, Form
from wtforms.fields import StringField, PasswordField, SubmitField
from wtforms.validators import Length, NumberRange, DataRequired, Email, ValidationError, EqualTo
from flask import request
app = Flask(__name__)
# 加载配置文件
app.config.from_object(config)
CORS(app, supports_credentials=True)
loginmanager = LoginManager()
loginmanager.init_app(app)
db = Db()
class UserProfileForm(Form):
nickname = StringField(
validators=[DataRequired(
message='昵称不能为空'), Length(6, 12)])
email = StringField(
validators=[DataRequired(),
Email(message='邮箱格式不合法'),
Length(6, 36)])
password = PasswordField(validators=[
DataRequired(),
Length(6, 128, message='密码长度不正确'),
EqualTo('confirm', message='两次输入密码不一致')
])
result = []
i = 0
for chapter in chapters:
url = chapter.xpath('a/@href')[0]
name = chapter.xpath('a/text()')[0]
novel_id = novel['id']
if url is not None and name is not None and url != '':
result.append([url, name, novel_id])
print(result)
db.insertMany(
'insert into gysw_chapter (`url`, `name`, `novel_id`) values (%s, %s, %s)',
result)
print('书本 %s，插入章节成功，共 %i 章' % (novel['book_name'], len(result)))
except Exception as e:
print(e)
print('书本 %s，插入章节失败' % (novel['book_name'], ))
# 章节批量存库操作
if __name__ == '__main__':
db = Db()
for classfiy in db.getClassify():
id = classfiy.id
list = getNovelsByClassifyId(id)
for novel in list:
getChapters(novel)
# list = getNovelsByClassifyId(1)
# getChapters(list[0])