def get_user_songs(self,
user_start,
user_count,
thread_count=20,
thread_inteval_time=5):
user_list = database_tool().select_list_limit(table="user",
start=user_start,
count=user_count)[1]
try:
success_count = 0
_user_ranklist_songs = user_ranklist_songs()
_playlist_songs = playlist_songs()
with ThreadPoolExecutor(thread_count) as executer:
future_list = []
for user in user_list:
future_rank_all = executer.submit(
_user_ranklist_songs.get_user_ranklist_songs, user[0],
config.rank_type_all, config.all_rank_max)
future_rank_week = executer.submit(
_user_ranklist_songs.get_user_ranklist_songs, user[0],
config.rank_type_week, config.week_rank_max)
future_playlist = executer.submit(
_playlist_songs.get_playlist_songs_by_user_id, user[0])
future_list.append(future_rank_all)
future_list.append(future_rank_week)
future_list.append(future_playlist)
time.sleep(thread_inteval_time)
for future in future_list:
if future.result()[0]:
success_count += 1
return True
except Exception as e:
logger.error("get_user_songs failed",
"error_type:{},error:{}".format(type(e), e))
def get_song_comments_total(self,
song_id,
comment_type=config.song_comments_type_hot):
"""
获取评论总数
:param song_id: 歌曲id
:param comment_type: 评论类型
:return: 评论总数
"""
# 获取+解析+存储数据 ----------------------------------------------------
try:
content = self.get_song_page_comments(song_id=song_id,
comment_type=comment_type,
is_get_comment_total=True)
if content[0]:
_database_tool = database_tool()
if comment_type == config.song_comments_type_hot:
_database_tool.update_song_hot_comment_count(
song_id=song_id, song_hot_comment_count=content[1])
elif comment_type == config.song_comments_type_default:
_database_tool.update_song_default_comment_count(
song_id=song_id, song_default_comment_count=content[1])
else:
return False, None
_database_tool.commit()
_database_tool.close()
return True, content[1]
except Exception as e:
logger.error(
"get_song_comments_total_count failed",
"song_id:{},comment_type:{},error_type:{},error:{}".format(
song_id, comment_type, type(e), e))
return False, None
def get_existed_favourite_list(user_id=1493673206):
# favourite_lists = database_tool().execute(
# sql="select * from recomend_favourite",execute_type=1, return_type=2
# )
# for target user
# user_id='1493673206'
favourite_lists = database_tool().execute(
sql="select * from recomend_favourite where user_id={}".format(
user_id),
execute_type=1,
return_type=2)
favourite_list_dict = {
"user": [],
"playlist": [],
"playlist_name": [],
"song_id": [],
"song_name": [],
"artist": [],
"tag_name": [],
"tag_count": []
}
if favourite_lists[0]:
for favourite in favourite_lists[1]:
favourite_list_dict["user"].append(favourite[0])
favourite_list_dict["playlist"].append(favourite[1])
favourite_list_dict["playlist_name"].append(favourite[2])
favourite_list_dict["song_id"].append(favourite[3])
favourite_list_dict["song_name"].append(favourite[4])
favourite_list_dict["artist"].append(favourite[5])
favourite_list_dict["tag_name"].append(favourite[6])
favourite_list_dict["tag_count"].append(favourite[7])
data_frame = pandas.DataFrame(data=favourite_list_dict)
return data_frame
def get_user_song_score(self, score_min=50):
"""
获取 用户-歌曲-评分 数据集
:param score_min:数据集中最小分数
:return: DatasetAutoFolds
"""
user_song_list = database_tool().execute(
sql="select user_id,song_id,score from user_song where score>{}".
format(score_min),
execute_type=1,
return_type=2)
# 数据集
user_song_dict = {"user": [], "song": [], "score": []}
if user_song_list[0]:
for user_song in user_song_list[1]:
user_song_dict["user"].append(user_song[0])
user_song_dict["song"].append(user_song[1])
user_song_dict["score"].append(user_song[2])
# 字典转化为pandas数据框
data_frame = pandas.DataFrame(data=user_song_dict)
# 设置评分标准
reader = Reader(rating_scale=(0, 1000))
# 从pandas数据框加载数据集
return Dataset.load_from_df(df=data_frame[["user", "song", "score"]],
reader=reader)
def code_favourite_list(user_id=1493673206):
favourite_lists = database_tool().execute(
sql=favourite_sql.format(user_id), execute_type=1, return_type=2)
favourite_list_dict = {
"user": [],
"playlist": [],
"playlist_name": [],
"song_id": [],
"song_name": [],
"artist": [],
"tag_name": [],
"tag_count": []
}
if favourite_lists[0]:
for favourite in favourite_lists[1]:
favourite_list_dict["user"].append(favourite[0])
favourite_list_dict["playlist"].append(favourite[1])
favourite_list_dict["playlist_name"].append(favourite[2])
favourite_list_dict["song_id"].append(favourite[3])
favourite_list_dict["song_name"].append(favourite[4])
favourite_list_dict["artist"].append(favourite[5])
favourite_list_dict["tag_name"].append(favourite[6])
favourite_list_dict["tag_count"].append(favourite[7])
data_frame = pandas.DataFrame(data=favourite_list_dict)
return data_frame
def get_existed_ranklist(user_id=1493673206):
ranklists = database_tool().execute(
sql="select * from recomend_ranklist where user_id={}".format(
user_id),
execute_type=1,
return_type=2)
ranklist_dict = {
"user": [],
"ranklist_id": [],
"song_id": [],
"song_score": [],
"song_name": [],
"artist": [],
"tag_name": [],
"tag_count": []
}
if ranklists[0]:
for ranklist in ranklists[1]:
ranklist_dict["user"].append(ranklist[0])
ranklist_dict["ranklist_id"].append(ranklist[1])
ranklist_dict["song_id"].append(ranklist[2])
ranklist_dict["song_score"].append(ranklist[3])
ranklist_dict["song_name"].append(ranklist[4])
ranklist_dict["artist"].append(ranklist[5])
ranklist_dict["tag_name"].append(ranklist[6])
ranklist_dict["tag_count"].append(ranklist[7])
data_frame = pandas.DataFrame(data=ranklist_dict)
return data_frame
def music_records_list():
user_lists = database_tool().execute(
sql="select distinct userid from recomend_musicrecord",
execute_type=1,
return_type=2)
user_id_list = []
for i in (user_lists[1]):
user_id_list.append(i[0])
return user_id_list
def get_neighbors_same_songs(self,
user_id,
user_neighbors,
score_min=50,
limit=200):
"""
返回近邻列表中与user_id相同的歌曲
:param user_id: 用户id
:param user_neighbors: 近邻列表
:param score_min:
:param limit:
:return:
"""
_database_tool = database_tool()
# user_id的歌曲列表
self_song_score_list = []
self_user_song_list = _database_tool.execute(
sql=
"select * from user_song where user_id={} and score>{} order by score desc limit {}"
.format(user_id, score_min, limit),
execute_type=1,
return_type=2)[1]
for self_user_song in self_user_song_list:
song = _database_tool.select_by_column(table="song",
column="song_id",
value=self_user_song[1])[1]
# score song_id song_name
self_song_score_list.append([self_user_song[2], song[0], song[1]])
# 所有近邻用户的歌曲列表
all_user_song_score_list = []
for user in user_neighbors:
# 某近邻用户的歌曲列表
user_song_score_list = []
user_song_list = _database_tool.execute(
sql=
"select * from user_song where user_id={} and score>{} order by score desc limit {}"
.format(user[0], score_min, limit),
execute_type=1,
return_type=2)[1]
for user_song in user_song_list:
song = _database_tool.select_by_column(table="song",
column="song_id",
value=user_song[1])[1]
# score song_id song_name
user_song_score_list.append([user_song[2], song[0], song[1]])
# 与self相同歌曲
same_song_score_list = [
i for i in self_song_score_list if i in user_song_score_list
]
all_user_song_score_list.append({
"user": user,
"all": user_song_score_list,
"same": same_song_score_list
})
return self_song_score_list, all_user_song_score_list
def get_playlist_songs(self, playlist_id):
_database_tool = database_tool()
_database_tool.insert_many_playlist([[playlist_id, '', 0, 0, '']])
_database_tool.commit()
_database_tool.close()
playlist_songs().get_playlist_songs_by_playlist_id(
playlist_id=playlist_id,
playlist_type=config.normal_playlist,
playlist_songs_max=sys.maxsize)
def get_song_comments(self, song_start, song_count):
song_list = database_tool().select_list_limit(table="song",
start=song_start,
count=song_count)
if song_list[0]:
for song in song_list[1]:
song_comments().get_song_comments_hot(
song_id=song[0],
song_comments_hot_max=1000,
thread_count=10,
thread_inteval_time=2)
def code_music_records(user_id=1493673206):
records_lists = database_tool().execute(
sql=musicrecord_sql.format(user_id), execute_type=1, return_type=2)
music_record_dict = {"userid": [], "song_id": [], "song_score": []}
if records_lists[0]:
for record in records_lists[1]:
music_record_dict["userid"].append(record[0])
music_record_dict["song_id"].append(record[1])
music_record_dict["song_score"].append(record[2])
data_frame = pandas.DataFrame(data=music_record_dict)
return data_frame
def get_existed_music_records(user_id=1493673206):
records_lists = database_tool().execute(
sql="select * from recomend_musicrecord where userid={}".format(
user_id),
execute_type=1,
return_type=2)
music_record_dict = {"user": [], "song_id": [], "song_score": []}
if records_lists[0]:
for record in records_lists[1]:
music_record_dict["user"].append(record[0])
music_record_dict["song_id"].append(record[1])
music_record_dict["song_score"].append(record[2])
data_frame = pandas.DataFrame(data=music_record_dict)
return data_frame
def get_song_comments(self, song_start, song_count):
"""
获取歌曲热门评论(包括用户表,歌手表)
:param song_start: 歌曲表位移
:param song_count: 歌曲表数目
:return:
"""
song_list = database_tool().select_list_limit(table="song",
start=song_start,
count=song_count)
if song_list[0]:
for song in song_list[1]:
song_comments().get_song_comments_hot(
song_id=song[0],
song_comments_hot_max=1000,
thread_count=10,
thread_inteval_time=2)
def parse_user_song_score(self, step=2000, end=sys.maxsize):
"""
计算 用户-歌曲表 中的评分
:param end:
:param step:
:return:
"""
try:
_database_tool = database_tool()
user_song_count = 0
while (user_song_count < end):
user_song_list = _database_tool.execute(
sql="select * from user_song where score=0 limit {}".
format(step),
execute_type=1,
return_type=2)
if user_song_list[0] and len(user_song_list[1]) != 0:
last_user_song_list = []
for user_song in user_song_list[1]:
last_user_song_list.append([
user_song[0], user_song[1],
int(user_song[3] * config.factor_rank_all_score +
user_song[4] * config.factor_rank_week_score +
user_song[5] *
config.factor_playlist_like_pop +
user_song[6] *
config.factor_playlist_create_pop +
user_song[7] *
config.factor_playlist_collect_pop)
])
_database_tool.insert_many_user_song_column(
column="score", data_list=last_user_song_list)
_database_tool.commit()
user_song_count += step
logger.debug("parse_user_song_score success",
"limit:{}".format(step))
else:
break
except Exception as e:
logger.error("parse_user_song_score fail",
"error_type:{},error:{}".format(type(e), e))
def get_neighbors(self, algo: AlgoBase, user_id, k):
"""
获取user的近邻列表
:param user_id:用户id
:return:
"""
_database_tool = database_tool()
# 内部id
user_id_inner = algo.trainset.to_inner_uid(user_id)
# 近邻内部id列表
neighbor_id_inner_list = algo.get_neighbors(iid=user_id_inner, k=k)
# 近邻外部id列表
neighbor_id_list = (algo.trainset.to_raw_uid(inner_id)
for inner_id in neighbor_id_inner_list)
# 近邻用户列表
user_neighbors = []
for neighbor_id in neighbor_id_list:
user_neighbors.append(
_database_tool.select_by_column(table="user",
column="user_id",
value=neighbor_id)[1])
return user_neighbors
def get_user_ranklist_songs(self,
user_id=config.user_id,
rank_type=config.rank_type,
rank_max=config.rank_max):
"""
获取某用户排行榜歌曲列表
:param user_id: 用户id
:param rank_max: 排行榜歌曲获取最大数
:return: status: 是否获取到歌曲
:return: 排行榜歌曲列表
"""
# 请求参数
_first_param = first_param().get_first_param_ranklist(
user_id=user_id, rank_type=rank_type)
# 请求数据 ----------------------------------------------------
json_data = None
try:
content = request_data().get_request_data(
first_param=_first_param[1], url=config.url_user_rank)
if content[0]:
if rank_type == config.rank_type_all:
json_data = json.loads(content[1])["allData"]
elif rank_type == config.rank_type_week:
json_data = json.loads(content[1])["weekData"]
else:
return False, None
except KeyError as e:
logger.warning(
"get_user_ranklist_songs get failed, Maybe the guy's ranklist is hidden,can you see it in the webpage ?",
"user_id:{},rank_type:{},ranklist_url:https://music.163.com/#/user/songs/rank?id={} ,error_type:{},error:{}"
.format(user_id, rank_type, user_id, type(e), e))
return False, None
except Exception as e:
logger.error(
"get_user_ranklist_songs get failed",
"user_id:{},rank_type:{},error_type:{},error:{}".format(
user_id, rank_type, type(e), e))
return False, None
# 解析数据 ----------------------------------------------------
ranklist_id = str(user_id) + "r" + str(rank_type)
song_success_count = 0
rank_list = []
user_rank_list = []
song_list = []
artist_list = []
artist_song_list = []
song_rank_list = []
user_song_list = []
try:
# ranklist_id rank_type rank_date
rank_list.append(
[ranklist_id, rank_type,
int(round(time.time() * 1000))])
# user_id ranklist_id
user_rank_list.append([user_id, ranklist_id])
while song_success_count < rank_max and song_success_count < len(
json_data):
song_id = json_data[song_success_count]["song"]["id"]
song_score = json_data[song_success_count]["score"]
print(json_data)
print('#############################')
print(json_data[song_success_count])
print('#############################')
print(song_score)
# song_id song_name
song_list.append(
[song_id, json_data[song_success_count]["song"]["name"]])
# song_id ranklist_id song_score
song_rank_list.append([song_id, ranklist_id, song_score])
# 多个歌手
artist_count = 0
for artist in json_data[song_success_count]["song"]["ar"]:
# artist_id artist_name
artist_list.append([artist["id"], artist["name"]])
# artist_id song_id sort
artist_song_list.append(
[artist["id"], song_id, artist_count])
artist_count += 1
# user_id song_id ranklist_all/week_score
user_song_list.append([user_id, song_id, song_score])
song_success_count += 1
except Exception as e:
logger.error(
"get_user_ranklist_songs parse failed",
"user_id:{},rank_type:{},error_type:{},song_success_count:{},error:{}"
.format(user_id, rank_type, song_success_count, type(e), e))
return False, None
# 存储数据 ----------------------------------------------------
try:
_database_tool = database_tool()
_database_tool.insert_many_ranklist(rank_list)
_database_tool.insert_many_song(song_list)
_database_tool.insert_many_artist(artist_list)
_database_tool.commit()
_database_tool.insert_many_user_ranklist(user_rank_list)
_database_tool.insert_many_song_ranklist(song_rank_list)
_database_tool.insert_many_artist_song(artist_song_list)
_database_tool.insert_many_user_song_column(
column="rank_all_score"
if rank_type == config.rank_type_all else "rank_week_score",
data_list=user_song_list)
_database_tool.commit()
_database_tool.close()
except Exception as e:
logger.error(
"get_user_ranklist_songs save failed",
"user_id:{},rank_type:{},song_count_success,error_type:{},error:{}"
.format(user_id, rank_type, song_success_count, type(e), e))
return False, None
logger.info(
"get_user_ranklist_songs success",
"user_id:{},rank_type:{},song_success_count:{}".format(
user_id, rank_type, song_success_count))
return True, (rank_list, song_list, artist_list, user_rank_list,
song_rank_list, artist_song_list)
data_list=user_song_list)
_database_tool.commit()
_database_tool.close()
except Exception as e:
logger.error(
"get_user_ranklist_songs save failed",
"user_id:{},rank_type:{},song_count_success,error_type:{},error:{}"
.format(user_id, rank_type, song_success_count, type(e), e))
return False, None
logger.info(
"get_user_ranklist_songs success",
"user_id:{},rank_type:{},song_success_count:{}".format(
user_id, rank_type, song_success_count))
return True, (rank_list, song_list, artist_list, user_rank_list,
song_rank_list, artist_song_list)
if __name__ == "__main__":
_database_tool = database_tool()
_database_tool.insert_many_user([[config.user_id, config.user_name]])
_database_tool.commit()
_database_tool.close()
user_ranklist_songs().get_user_ranklist_songs(
user_id=config.user_id, rank_type=config.rank_type_all)
user_ranklist_songs().get_user_ranklist_songs(
user_id=config.user_id, rank_type=config.rank_type_week)
# user_id_list = database_tool().select_user_list(table='user')
# user_ranklist_songs().get_user_ranklist_songs_thread(user_list=user_id_list[1], thread_count=5,
# thread_inteval_time=2,
# rank_max=300)
def get_user_playlists(
self,
user_id=config.user_id,
created_playlists_max=config.created_playlists_max,
collected_playlists_max=config.collected_playlists_max,
is_playlists_default=config.is_playlists_default,
is_playlists_created=config.is_playlists_created,
is_playlists_collected=config.is_playlists_collected):
"""
获取用户歌单列表
:param user_id: 用户id
:param created_playlists_max: 用户创建的歌单获取最大数
:param collected_playlists_max: 用户收集的歌单获取最大数
:param is_playlists_default: 是否爬取”用户喜欢的音乐“歌单
:param is_playlists_created: 是否爬取用户创建的歌单
:param is_playlists_collected: 是否爬取用户收藏的歌单
:return: status: 是否获取到歌单
:return: 歌单列表
:return: 用户歌单列表
"""
# 请求参数 ----------------------------------------------------
_first_param = first_param().get_first_param_user_playlists(
user_id=user_id)
# 请求数据 ----------------------------------------------------
try:
content = request_data().get_request_data(
_first_param[1], url=config.url_user_playlists)
if content[0]:
json_playlists_data = json.loads(content[1])["playlist"]
else:
return False, None
except Exception as e:
logger.error(
"get_user_playlists get failed",
"user_id:{},error_type:{},error:{}".format(
user_id, type(e), e))
return False, None
# 解析数据 ----------------------------------------------------
playlist_count = 0
created_playlists_count = 0
collected_playlists_count = 0
playlist_list = []
user_playlists_list = []
tag_list = []
playlist_tag_list = []
try:
while playlist_count < len(json_playlists_data):
# ”用户喜欢的音乐“歌单
if is_playlists_default:
parse = self.__parser(
user_id=user_id,
playlist_count=playlist_count,
data=json_playlists_data,
playlist_type=config.default_playlist)
playlist_list.append(parse["playlist_list"])
user_playlists_list.append(parse["user_playlist_list"])
if len(parse["tag_list"]) != 0:
tag_list.extend(parse["tag_list"])
playlist_tag_list.extend(parse["playlist_tag_list"])
playlist_count += 1
is_playlists_default = False
continue
# 用户创建的歌单
elif is_playlists_created:
# 若此处出现playlist为0的情况，说明没有爬取“用户喜欢的音乐”歌单，跳过此歌单
if playlist_count == 0:
playlist_count += 1
continue
# 是否是用户自己创建的歌单
elif str(json_playlists_data[playlist_count]["creator"]
['userId']) == str(user_id):
if created_playlists_count < created_playlists_max:
parse = self.__parser(
user_id=user_id,
playlist_count=playlist_count,
data=json_playlists_data,
playlist_type=config.created_playlist)
playlist_list.append(parse["playlist_list"])
user_playlists_list.append(
parse["user_playlist_list"])
if len(parse["tag_list"]) != 0:
tag_list.extend(parse["tag_list"])
playlist_tag_list.extend(
parse["playlist_tag_list"])
created_playlists_count += 1
playlist_count += 1
else:
is_playlists_created = False
continue
# 用户收藏的歌单
elif is_playlists_collected:
# 是否是用户收藏的歌单
if str(json_playlists_data[playlist_count]["creator"]
['userId']) != str(user_id):
if collected_playlists_count < collected_playlists_max:
parse = self.__parser(
user_id=user_id,
playlist_count=playlist_count,
data=json_playlists_data,
playlist_type=config.collected_playlist)
playlist_list.append(parse["playlist_list"])
user_playlists_list.append(
parse["user_playlist_list"])
if len(parse["tag_list"]) != 0:
tag_list.extend(parse["tag_list"])
playlist_tag_list.extend(
parse["playlist_tag_list"])
collected_playlists_count += 1
# 若此处出现不是用户收藏的歌单情况，说明用户创建歌单没有完全爬取，继续循环
playlist_count += 1
continue
break
except Exception as e:
logger.error(
"get_user_playlists parse failed",
"user_id:{},created_playlists_count:{},collected_playlists_count:{},error_type:{},error:{}"
.format(user_id, created_playlists_count,
collected_playlists_count, type(e), e))
return False, None
# 存储数据 ----------------------------------------------------
try:
_database_tool = database_tool()
_database_tool.insert_many_playlist(playlist_list)
_database_tool.insert_many_tag(tag_list)
_database_tool.commit()
_database_tool.insert_many_user_playlist(user_playlists_list)
_database_tool.insert_many_playlist_tag(playlist_tag_list)
_database_tool.commit()
_database_tool.close()
except Exception as e:
logger.error(
"get user_playlist save failed",
"user_id:{},created_playlists_count:{},collected_playlists_count:{},error_type:{},error:{}"
.format(user_id, created_playlists_count,
collected_playlists_count, type(e), e))
return False, None
logger.info(
"get user_playlist success",
"user_id:{},created_playlists_count:{},collected_playlists_count:{}"
.format(user_id, created_playlists_count,
collected_playlists_count))
return True, playlist_list, user_playlists_list
data[playlist_count]["id"], data[playlist_count]["name"],
data[playlist_count]["trackCount"],
data[playlist_count]["playCount"],
data[playlist_count]["updateTime"]
],
"user_playlist_list":
[user_id, data[playlist_count]["id"], playlist_type],
"tag_list":
tag_list,
"playlist_tag_list":
playlist_tag_list
}
if __name__ == "__main__":
_database_tool = database_tool()
_database_tool.insert_many_user([[config.user_id, config.user_name]])
_database_tool.commit()
_database_tool.close()
max = sys.maxsize
# 爬取 喜欢歌单
# print(user_playlists().get_user_playlists(config.user_id, created_playlists_max=max, collected_playlists_max=max,
# is_playlists_default=True, is_playlists_created=False,
# is_playlists_collected=False))
# # 爬取 创建歌单
# print(user_playlists().get_user_playlists(config.user_id, created_playlists_max=max, collected_playlists_max=max,
# is_playlists_default=False, is_playlists_created=True,
# is_playlists_collected=False))
# print(user_playlists().get_user_playlists(config.user_id, created_playlists_max=3, collected_playlists_max=max,
# is_playlists_default=False, is_playlists_created=True,
def get_playlist_songs_by_playlist_id(self, playlist_id=config.playlist_id, user_id=0,
playlist_type=config.playlist_type,
playlist_songs_max=config.playlist_songs_max):
"""
通过歌单id获取歌单歌曲列表
:param playlist_id: 歌单id
:param user_id: 用户id
:param playlist_type: 歌单类型，包括 default created collected
:param playlist_songs_max: 歌单最大歌曲选取数
:return: status: 是否获取到歌曲
:return: 歌曲个数
:return: 歌曲列表
:return: 歌手列表
:return: 歌曲歌手列表
"""
# 请求参数 ----------------------------------------------------
_first_param = first_param().get_first_param_playlist(playlist_id)
# 请求数据 ----------------------------------------------------
try:
content = request_data().get_request_data(first_param=_first_param[1], url=config.url_playlist)
if content[0]:
songs_data = json.loads(content[1])["playlist"]["tracks"]
tags_data = json.loads(content[1])["playlist"]["tags"]
creator_data = json.loads(content[1])["playlist"]["creator"]
else:
return False, None
except Exception as e:
logger.error("get_playlist_songs_by_playlist_id get failed",
"playlist_id:{},playlist_type:{},error_type:{},error:{}"
.format(playlist_id, playlist_type, type(e), e))
return False, None
# 解析数据 ----------------------------------------------------
song_count = 0
song_list = []
artist_list = []
artist_song_list = []
song_playlist_list = []
tag_list = tags_data
song_tag_list = []
# user_id user_name
creator_list = [[
creator_data["userId"],
creator_data["nickname"]
]]
# user_id playlist_id playlist_type
creator_playlist_list = [[
creator_data["userId"],
playlist_id,
config.created_playlist
]]
# 创作者-歌曲列表
creator_song_list = []
user_song_list = []
try:
while song_count < playlist_songs_max and song_count < len(songs_data):
song_id = songs_data[song_count]["id"]
song_pop = int(songs_data[song_count]["pop"])
# song_id song_name
song_list.append([
song_id,
songs_data[song_count]["name"]
])
# 多个歌手
artist_count = 0
for artist in songs_data[song_count]['ar']:
# artist_id artist_name
artist_list.append([
artist["id"],
artist["name"]
])
# artist_id song_id sort
artist_song_list.append([
artist["id"],
song_id,
artist_count
])
artist_count += 1
# song_id playlist_id song_pop playlist_type
song_playlist_list.append([
song_id,
playlist_id,
song_pop,
playlist_type
])
# song_id tag_name
for tag in tag_list:
song_tag_list.append([
song_id,
tag
])
# user_id song_id playlist_like/create/collect_pop
user_song_list.append([
user_id,
song_id,
song_pop
])
# user_id song_id playlist_create_pop
creator_song_list.append([
creator_data["userId"],
song_id,
song_pop
])
song_count += 1
except Exception as e:
logger.error("get_playlist_songs_by_playlist_id parse failed",
"playlist_id:{},playlist_type:{},song_count:{},error_type:{},error:{}"
.format(playlist_id, playlist_type, song_count, type(e), e))
return False, None
# 存储数据 ----------------------------------------------------
try:
_database_tool = database_tool()
_database_tool.insert_many_song(song_list)
_database_tool.insert_many_artist(artist_list)
_database_tool.insert_many_tag(tag_list)
_database_tool.insert_many_user(creator_list)
_database_tool.commit()
_database_tool.insert_many_song_playlist(song_playlist_list)
_database_tool.insert_many_artist_song(artist_song_list)
_database_tool.insert_many_song_tag(song_tag_list)
_database_tool.insert_many_user_playlist(creator_playlist_list)
_database_tool.insert_many_user_song_column(column="playlist_create_pop", data_list=creator_song_list)
if playlist_type == config.default_playlist:
column = "playlist_like_pop"
elif playlist_type == config.created_playlist:
column = "playlist_create_pop"
elif playlist_type == config.collected_playlist:
column = "playlist_collect_pop"
else:
column = ""
if column != "":
_database_tool.insert_many_user_song_column(column=column, data_list=user_song_list)
_database_tool.commit()
_database_tool.close()
except Exception as e:
logger.error("get_playlist_songs_by_playlist_id save failed",
"playlist_id:{},playlist_type:{},error_type:{},error:{}"
.format(playlist_id, playlist_type, type(e), e))
return False, None
logger.info("get_playlist_songs_by_playlist_id success",
"playlist_id:{},playlist_type:{},song_count:{}"
.format(playlist_id, playlist_type, song_count))
return True, len(song_list), song_list, artist_list, artist_song_list
def get_song_page_comments(self,
song_id,
comment_type=config.song_comments_type_hot,
offset=0,
limit=0,
is_get_comment_total=False):
"""
通过song_id获取某页评论
:param song_id: 歌曲id
:param comment_type: 评论类型,详见config
:param offset: 位移量
:param limit: 每页评论限制量
:param is_get_comment_total: 是否为获取评论总数，默认否
:return: 评论数据
"""
# 请求参数 ----------------------------------------------------
_first_param = first_param().get_first_param_eapi_comment(
offset=offset, limit=limit)
if comment_type == config.song_comments_type_hot:
url = config.get_comments_hot_url(song_id)
eapi_url = config.get_comment_hot_url_for_eapi_param(song_id)
elif comment_type == config.song_comments_type_default:
url = config.get_comments_default_url(song_id)
eapi_url = config.get_comment_default_url_for_eapi_param(song_id)
else:
return False, None
# 请求数据 ----------------------------------------------------
try:
content = request_data().get_request_data(
first_param=_first_param[1],
url=url,
api_type=config.api_eapi,
eapi_url=eapi_url)
if content[0]:
if is_get_comment_total:
return True, json.loads(content[1])["total"]
elif comment_type == config.song_comments_type_default:
json_data = json.loads(content[1])["comments"]
elif comment_type == config.song_comments_type_hot:
json_data = json.loads(content[1])["hotComments"]
else:
return False, None
else:
return False, None
except Exception as e:
logger.error(
"get_song_page_comments get failed",
"song_id:{},comment_type:{},offset:{},limit:{},error_type:{},error:{}"
.format(song_id, comment_type, offset, limit, type(e), e))
return False, []
# 解析数据 ----------------------------------------------------
user_list = []
comment_list = []
user_comment_list = []
song_comment_list = []
try:
for comment_json in json_data:
# user_id user_name
user_list.append([
comment_json["user"]["userId"],
comment_json["user"]["nickname"]
])
# comment_id comment_type comment_date comment_content comment_like_count
comment_list.append([
comment_json["commentId"], comment_type,
comment_json["time"], comment_json["content"],
comment_json["likedCount"]
])
# user_id comment_id
user_comment_list.append([
comment_json["user"]["userId"], comment_json["commentId"]
])
# song_id comment_id
song_comment_list.append([song_id, comment_json["commentId"]])
# logger.info("get_song_page_comments parse success",
# "song_id:{},comment_total:{},comment_type:{},offset:{},limit:{}"
# .format(song_id, len(comment_list), comment_type, offset, limit))
except Exception as e:
logger.error(
"get_song_page_comments parse failed",
"song_id:{},comment_type:{},offset:{},limit:{},error_type:{},error:{}"
.format(song_id, comment_type, offset, limit, type(e), e))
# 存储数据 ----------------------------------------------------
try:
_database_tool = database_tool()
_database_tool.insert_many_user(user_list)
_database_tool.insert_many_comment(comment_list)
_database_tool.commit()
_database_tool.insert_many_song_comment(song_comment_list)
_database_tool.insert_many_user_comment(user_comment_list)
_database_tool.commit()
_database_tool.close()
except Exception as e:
logger.error(
"get_song_page_comments save failed",
"song_id:{},comment_type:{},offset:{},limit:{},error_type:{},error:{}"
.format(song_id, comment_type, offset, limit, type(e), e))
return True, (user_list, comment_list, user_comment_list)
