def prser_news_content(news, data):
docid = news.get("docid")
if data:
data = data.get(docid)
else:
return
body = data.get("body")
sql = Sql()
ret = sql.is_exists(news, "docid")
if ret == 0:
# 去除视频标签
body = re.sub("\<!--VIDEO#\d+\-\-\>", "", body)
body = re.sub("<p>\s+<b>【欢迎搜索关注公众号“足球大会”.*", "", body)
# 将图片标签替换成静态图片
for i, img in enumerate(data.get("img"), 0):
src = img.get("src")
digit = hash(data.get("docid"))
_, s = divmod(digit, 26)
dir = chr(s + 97)
base = cf.get("path", "news")
path = os.path.join(base, dir,
docid + f"_{i}." + src.split(".")[-1])
img_path = os.path.join("img/news", dir,
docid + f"_{i}." + src.split(".")[-1])
body = body.replace(img.get("ref"), f'<img src="{img_path}">')
img_content = get_content_data(img.get("src"))
save_pic(path, img_content)
news["content"] = body
sql.save(news)
sql.close()
def parse_hot_expert(data, status, index):
expert = dict(table="hot_expert")
expert["id"] = data.get("userId")
expert["popularity"] = data.get("popularity")
expert["earning_rate"] = data.get("earningRate")
expert["type"] = status
expert["top_index"] = index
sql = Sql()
sql.save(expert)
def parse_match_list(match_list, article_id):
sql = Sql()
for m in match_list:
match = dict(table="matches")
# print(json.dumps(m,ensure_ascii=False))
match["category_id"] = m.get("categoryId")
match["category_name"] = m.get("categoryName")
match["info_id"] = m.get("matchInfoId")
match["match_status"] = m.get("matchStatus")
if match.get("match_status") == 3:
match["status"] = "完"
else:
match["status"] = "未"
# match_time = m.get("matchTime")
match_time = m.get("matchTimeAc")
match_time = match_time.replace("/", "-").replace("/", "-")
if not re.search("\d{4}", match_time):
match_time = str(datetime.date.today().year) + "-" + match_time
match["match_time"] = match_time
league = dict(table="leaguematch")
league_id = m.get("leagueId")
league_name = m.get("leagueName")
league["id"] = league_id
league["name"] = league_name
sql.save_if_not_exist(league)
match["league_id"] = league_id
match["league_name"] = league_name
g1 = m.get("guestTeam")
parse_team(g1)
g2 = m.get("homeTeam")
parse_team(g2)
match["guest_name"] = g1.get("teamName")
match["home_name"] = g2.get("teamName")
match["guest_id"] = g1.get("teamId")
match["home_id"] = g2.get("teamId")
# match["guest_name"] = m.get("guestName")
match["guest_score"] = m.get("guestScore")
# match["home_name"] = m.get("homeName")
match["home_score"] = m.get("homeScore")
sql.save_if_not_exist(match, "info_id")
article_match = dict(table="article_match")
article_match["article_id"] = article_id
article_match["info_id"] = match["info_id"]
print("article_match", article_match)
if not sql.is_exists_by_tow(article_match, "article_id", "info_id"):
sql.save(article_match)
sql.close()
def get_expert_league_info(expert_id):
url = cf.get("api", "expert_league_info")
url = url.replace("@", str(expert_id))
data = get_json_data(url)
if not data:
return
data = data.get("data")
for m in data:
sql = Sql()
expert_league = dict(table="expert_leaguematches")
expert_league["expert_id"] = expert_id
league_id = m.get("leagueMatchId")
expert_league["leaguematch_id"] = league_id
expert_league["leaguematch_name"] = m.get("leagueMatchName")
expert_league["best_hitrate"] = m.get("bestMatchesHitRate")
expert_league["hitrate_desc"] = m.get("totalHitRateDesc")
if not sql.is_exists_by_tow(expert_league, "expert_id", "leaguematch_id"):
sql.save(expert_league)
print("save")
sql.close()
get_expert_league_articles(expert_id, league_id)
def get_expert_league_articles(expert_id, league_id):
url = cf.get("api", "expert_league_articles")
url = url.replace("expertid", str(expert_id)).replace("leagueid", str(league_id))
data = get_json_data(url).get("data").get("threadList")
print(json.dumps(data, ensure_ascii=False))
for a in data:
sql = Sql()
print(json.dumps(a, ensure_ascii=False))
article = dict(table="articles")
article["id"] = a.get("threadId")
article["title"] = a.get("threadTitle")
article["expert_id"] = expert_id
article["lottery_category_id"] = a.get("lotteryCategoryId")
article["lottery_category_name"] = a.get("lotteryCategoryName")
article["is_win"] = a.get("isWin")
article["publish_time"] = a.get("publishTime")
article["price"] = a.get("price")
article["league_id"] = league_id
for m in a.get("matchList"):
match = dict(table="matches")
match["category_id"] = m.get("categoryId")
match["category_name"] = m.get("categoryName")
match["info_id"] = m.get("matchInfoId")
match["match_status"] = m.get("matchStatus")
if match.get("match_status") == 3:
match["status"] = "完"
else:
match["status"] = "未"
match_time = m.get("matchTime")
match_time = match_time.replace("/", "-").replace("/", "-")
if not re.search("\d{4}", match_time):
match_time = str(datetime.date.today().year) + "-" + match_time
print(match_time)
match["match_time"] = match_time
league = dict(table="leaguematch")
league_id = m.get("leagueId")
league_name = m.get("leagueName")
league["id"] = league_id
league["name"] = league_name
sql.save_if_not_exist(league)
match["league_id"] = league_id
match["league_name"] = league_name
match["guest_name"] = m.get("guestName")
match["guest_score"] = m.get("guestScore")
match["home_name"] = m.get("homeName")
match["home_score"] = m.get("homeScore")
sql.save_if_not_exist(match, "info_id")
article_match = dict(table="article_match")
article_match["article_id"] = article["id"]
article_match["info_id"] = match["info_id"]
if not sql.is_exists_by_tow(article_match, "article_id", "info_id"):
sql.save(article_match)
ret = sql.save_if_not_exist(article)
if ret == 0:
article["table"] = "articles"
sql.update(article, "league_id")
sql.close()