def add_news_detail(title=None,
category=None,
img=None,
time=None,
url=None,
content=None,
artical_id=None,
html=None,
excerpt=None):
news = News_Model(title=title,
category=category,
img=img,
time=time,
url=url,
artical_id=artical_id,
content=content,
html=html,
excerpt=excerpt)
result = news.query.filter_by(artical_id=artical_id).all()
if (result == None or len(result) == 0):
from app.main import log
log.info("add News_Model %s" % title)
try:
alchemy_db.session.add(news)
alchemy_db.session.commit()
pass
except BaseException as e:
log.error("add News_Model error %s" % e)
alchemy_db.session.rollback()
# alchemy_db.session.close()
pass
finally:
pass
async def request_index():
'''先分析首页 豆瓣新片榜 再拿旁边的 分类'''
html = await request(host_chart, decode=False, fr="豆瓣首页")
if html != None:
try:
await prase_index(html)
pass
except BaseException as e:
log.info(e)
log.info(html)
pass
pass
def testAPI(api_names):
if api_names == "get_news_cat":
from app.db import news_db
result_list = news_db.News_Category.get_all(2)
# json_data = toJson(result_list)
if api_names == "get_news_model":
from app.db import news_db
result_list = news_db.News_Model.get_all(10)
# json_data = toJson(result_list)
base_model = Base_Model("success", 0, data=result_list)
json = toJson(base_model)
log.info("result: " + json)
def add_movie_comment(content, movie_id):
comment = Movie_Comment(content, movie_id)
from app.main import log
log.info("add_comment comment %s" % content)
try:
alchemy_db.session.add(comment)
alchemy_db.session.commit()
pass
except BaseException as e:
log.error(" add_movie_comment error %s" % e)
alchemy_db.session.rollback()
# alchemy_db.session.close()
pass
def add_news_cate(type, category_id, category_name, artical_id):
cat = News_Category(type, category_id, category_name, artical_id)
if len(cat.query.filter_by(artical_id=artical_id).all()) == 0:
from app.main import log
log.info("add News_Category %s artical_id %s" %
(category_name, artical_id))
try:
alchemy_db.session.add(cat)
alchemy_db.session.commit()
pass
except BaseException as e:
log.error("add News_Category error %s" % e)
alchemy_db.session.rollback()
# alchemy_db.session.close()
pass
def add_news_img(artical_id, url, desc):
img = News_Img(artical_id, url, desc)
if len(img.query.filter_by(url=url).all()) == 0 or img.query.filter_by(
url=url).all() == None:
from app.main import log
log.info("add add_news_img des %s" % desc)
try:
alchemy_db.session.add(img)
alchemy_db.session.commit()
pass
except BaseException as e:
log.error("add News_Img error %s" % e)
alchemy_db.session.rollback()
# alchemy_db.session.close()
pass
async def prase_index(html):
'''先分析首页 豆瓣新片榜 再拿旁边的 分类'''
soup = BeautifulSoup(html, "html.parser")
divs = soup.find("div", class_="indent")
tables = divs.find_all("table")
for table in tables:
# https://movie.douban.com/subject/26811587/
a = table.find("a")
url = a["href"]
url = url[:url.__len__() - 1]
dot_index = url.rindex("/")
id = url[dot_index + 1:]
log.info("movie id %s " % (id))
# await requestDetail(id)
await request_comment(id, 0)
pass
pass
def handle_api(api_names,args = None,form=None,data = None,request = None):
ip = request.remote_addr
log.info("ip "+ip)
json_data = None
if api_names == "get_news_list":
json_data = handle_get_news_list()
if api_names == "get_pic":
json_data = handle_pic()
if api_names == "get_duanzi":
page_size = args.get('page_size')
page_num = args.get('page_num')
json_data = hanle_api_pengfu(page_size,page_num)
if api_names == "refresh_news":
from app.pic_spider import qdaily
qdaily.refresh_news()
json_data = "refresh ok"
if api_names == "create_db":
from app.db import news_db
news_db.create_db()
json_data = "create ok"
if api_names == "get_news_cat":
from app.db import news_db
result_list = news_db.News_Category.get_all(2)
json_data = toJson(result_list)
if api_names == "get_news_model":
from app.db import news_db
result_list = news_db.News_Model.get_all(2)
json_data = toJson(result_list)
# from app.data_model.base_model import base_model
# log.info("data " + str(json_data))
base_model = Base_Model("success",0,json_data)
json = toJson(base_model)
log.info("result: " + json)
return json
async def parse_comment(html, id):
soup = BeautifulSoup(html, "html.parser")
div_article = soup.find("div", class_="article")
num_span = div_article.find("span", id="collections_bar")
num_str = num_span.get_text() #118732人看过
lenth = "人看过".__len__()
num = int(num_str[0:-lenth])
total_comment = num
sub_ins = div_article.find("div", class_="sub_ins")
tables = sub_ins.find_all("table")
for table in tables:
p = table.find_all("p")
for p1 in p:
p_str = p1.get_text()
log.info(p_str)
from app.db.movie_db import Movie_Comment
comment = Movie_Comment(p_str, id)
Movie_Comment.add_movie_comment(p_str, id)
pass
pass
pass
def info(msg):
log.info(msg)