def get_user_songs(self, user_start, user_count, thread_count=20, thread_inteval_time=5): user_list = database_tool().select_list_limit(table="user", start=user_start, count=user_count)[1] try: success_count = 0 _user_ranklist_songs = user_ranklist_songs() _playlist_songs = playlist_songs() with ThreadPoolExecutor(thread_count) as executer: future_list = [] for user in user_list: future_rank_all = executer.submit( _user_ranklist_songs.get_user_ranklist_songs, user[0], config.rank_type_all, config.all_rank_max) future_rank_week = executer.submit( _user_ranklist_songs.get_user_ranklist_songs, user[0], config.rank_type_week, config.week_rank_max) future_playlist = executer.submit( _playlist_songs.get_playlist_songs_by_user_id, user[0]) future_list.append(future_rank_all) future_list.append(future_rank_week) future_list.append(future_playlist) time.sleep(thread_inteval_time) for future in future_list: if future.result()[0]: success_count += 1 return True except Exception as e: logger.error("get_user_songs failed", "error_type:{},error:{}".format(type(e), e))
def get_song_comments_total(self, song_id, comment_type=config.song_comments_type_hot): """ 获取评论总数 :param song_id: 歌曲id :param comment_type: 评论类型 :return: 评论总数 """ # 获取+解析+存储数据 ---------------------------------------------------- try: content = self.get_song_page_comments(song_id=song_id, comment_type=comment_type, is_get_comment_total=True) if content[0]: _database_tool = database_tool() if comment_type == config.song_comments_type_hot: _database_tool.update_song_hot_comment_count( song_id=song_id, song_hot_comment_count=content[1]) elif comment_type == config.song_comments_type_default: _database_tool.update_song_default_comment_count( song_id=song_id, song_default_comment_count=content[1]) else: return False, None _database_tool.commit() _database_tool.close() return True, content[1] except Exception as e: logger.error( "get_song_comments_total_count failed", "song_id:{},comment_type:{},error_type:{},error:{}".format( song_id, comment_type, type(e), e)) return False, None
def get_existed_favourite_list(user_id=1493673206): # favourite_lists = database_tool().execute( # sql="select * from recomend_favourite",execute_type=1, return_type=2 # ) # for target user # user_id='1493673206' favourite_lists = database_tool().execute( sql="select * from recomend_favourite where user_id={}".format( user_id), execute_type=1, return_type=2) favourite_list_dict = { "user": [], "playlist": [], "playlist_name": [], "song_id": [], "song_name": [], "artist": [], "tag_name": [], "tag_count": [] } if favourite_lists[0]: for favourite in favourite_lists[1]: favourite_list_dict["user"].append(favourite[0]) favourite_list_dict["playlist"].append(favourite[1]) favourite_list_dict["playlist_name"].append(favourite[2]) favourite_list_dict["song_id"].append(favourite[3]) favourite_list_dict["song_name"].append(favourite[4]) favourite_list_dict["artist"].append(favourite[5]) favourite_list_dict["tag_name"].append(favourite[6]) favourite_list_dict["tag_count"].append(favourite[7]) data_frame = pandas.DataFrame(data=favourite_list_dict) return data_frame
def get_user_song_score(self, score_min=50): """ 获取 用户-歌曲-评分 数据集 :param score_min:数据集中最小分数 :return: DatasetAutoFolds """ user_song_list = database_tool().execute( sql="select user_id,song_id,score from user_song where score>{}". format(score_min), execute_type=1, return_type=2) # 数据集 user_song_dict = {"user": [], "song": [], "score": []} if user_song_list[0]: for user_song in user_song_list[1]: user_song_dict["user"].append(user_song[0]) user_song_dict["song"].append(user_song[1]) user_song_dict["score"].append(user_song[2]) # 字典转化为pandas数据框 data_frame = pandas.DataFrame(data=user_song_dict) # 设置评分标准 reader = Reader(rating_scale=(0, 1000)) # 从pandas数据框加载数据集 return Dataset.load_from_df(df=data_frame[["user", "song", "score"]], reader=reader)
def code_favourite_list(user_id=1493673206): favourite_lists = database_tool().execute( sql=favourite_sql.format(user_id), execute_type=1, return_type=2) favourite_list_dict = { "user": [], "playlist": [], "playlist_name": [], "song_id": [], "song_name": [], "artist": [], "tag_name": [], "tag_count": [] } if favourite_lists[0]: for favourite in favourite_lists[1]: favourite_list_dict["user"].append(favourite[0]) favourite_list_dict["playlist"].append(favourite[1]) favourite_list_dict["playlist_name"].append(favourite[2]) favourite_list_dict["song_id"].append(favourite[3]) favourite_list_dict["song_name"].append(favourite[4]) favourite_list_dict["artist"].append(favourite[5]) favourite_list_dict["tag_name"].append(favourite[6]) favourite_list_dict["tag_count"].append(favourite[7]) data_frame = pandas.DataFrame(data=favourite_list_dict) return data_frame
def get_existed_ranklist(user_id=1493673206): ranklists = database_tool().execute( sql="select * from recomend_ranklist where user_id={}".format( user_id), execute_type=1, return_type=2) ranklist_dict = { "user": [], "ranklist_id": [], "song_id": [], "song_score": [], "song_name": [], "artist": [], "tag_name": [], "tag_count": [] } if ranklists[0]: for ranklist in ranklists[1]: ranklist_dict["user"].append(ranklist[0]) ranklist_dict["ranklist_id"].append(ranklist[1]) ranklist_dict["song_id"].append(ranklist[2]) ranklist_dict["song_score"].append(ranklist[3]) ranklist_dict["song_name"].append(ranklist[4]) ranklist_dict["artist"].append(ranklist[5]) ranklist_dict["tag_name"].append(ranklist[6]) ranklist_dict["tag_count"].append(ranklist[7]) data_frame = pandas.DataFrame(data=ranklist_dict) return data_frame
def music_records_list(): user_lists = database_tool().execute( sql="select distinct userid from recomend_musicrecord", execute_type=1, return_type=2) user_id_list = [] for i in (user_lists[1]): user_id_list.append(i[0]) return user_id_list
def get_neighbors_same_songs(self, user_id, user_neighbors, score_min=50, limit=200): """ 返回近邻列表中与user_id相同的歌曲 :param user_id: 用户id :param user_neighbors: 近邻列表 :param score_min: :param limit: :return: """ _database_tool = database_tool() # user_id的歌曲列表 self_song_score_list = [] self_user_song_list = _database_tool.execute( sql= "select * from user_song where user_id={} and score>{} order by score desc limit {}" .format(user_id, score_min, limit), execute_type=1, return_type=2)[1] for self_user_song in self_user_song_list: song = _database_tool.select_by_column(table="song", column="song_id", value=self_user_song[1])[1] # score song_id song_name self_song_score_list.append([self_user_song[2], song[0], song[1]]) # 所有近邻用户的歌曲列表 all_user_song_score_list = [] for user in user_neighbors: # 某近邻用户的歌曲列表 user_song_score_list = [] user_song_list = _database_tool.execute( sql= "select * from user_song where user_id={} and score>{} order by score desc limit {}" .format(user[0], score_min, limit), execute_type=1, return_type=2)[1] for user_song in user_song_list: song = _database_tool.select_by_column(table="song", column="song_id", value=user_song[1])[1] # score song_id song_name user_song_score_list.append([user_song[2], song[0], song[1]]) # 与self相同歌曲 same_song_score_list = [ i for i in self_song_score_list if i in user_song_score_list ] all_user_song_score_list.append({ "user": user, "all": user_song_score_list, "same": same_song_score_list }) return self_song_score_list, all_user_song_score_list
def get_playlist_songs(self, playlist_id): _database_tool = database_tool() _database_tool.insert_many_playlist([[playlist_id, '', 0, 0, '']]) _database_tool.commit() _database_tool.close() playlist_songs().get_playlist_songs_by_playlist_id( playlist_id=playlist_id, playlist_type=config.normal_playlist, playlist_songs_max=sys.maxsize)
def get_song_comments(self, song_start, song_count): song_list = database_tool().select_list_limit(table="song", start=song_start, count=song_count) if song_list[0]: for song in song_list[1]: song_comments().get_song_comments_hot( song_id=song[0], song_comments_hot_max=1000, thread_count=10, thread_inteval_time=2)
def code_music_records(user_id=1493673206): records_lists = database_tool().execute( sql=musicrecord_sql.format(user_id), execute_type=1, return_type=2) music_record_dict = {"userid": [], "song_id": [], "song_score": []} if records_lists[0]: for record in records_lists[1]: music_record_dict["userid"].append(record[0]) music_record_dict["song_id"].append(record[1]) music_record_dict["song_score"].append(record[2]) data_frame = pandas.DataFrame(data=music_record_dict) return data_frame
def get_existed_music_records(user_id=1493673206): records_lists = database_tool().execute( sql="select * from recomend_musicrecord where userid={}".format( user_id), execute_type=1, return_type=2) music_record_dict = {"user": [], "song_id": [], "song_score": []} if records_lists[0]: for record in records_lists[1]: music_record_dict["user"].append(record[0]) music_record_dict["song_id"].append(record[1]) music_record_dict["song_score"].append(record[2]) data_frame = pandas.DataFrame(data=music_record_dict) return data_frame
def get_song_comments(self, song_start, song_count): """ 获取歌曲热门评论(包括用户表,歌手表) :param song_start: 歌曲表位移 :param song_count: 歌曲表数目 :return: """ song_list = database_tool().select_list_limit(table="song", start=song_start, count=song_count) if song_list[0]: for song in song_list[1]: song_comments().get_song_comments_hot( song_id=song[0], song_comments_hot_max=1000, thread_count=10, thread_inteval_time=2)
def parse_user_song_score(self, step=2000, end=sys.maxsize): """ 计算 用户-歌曲表 中的评分 :param end: :param step: :return: """ try: _database_tool = database_tool() user_song_count = 0 while (user_song_count < end): user_song_list = _database_tool.execute( sql="select * from user_song where score=0 limit {}". format(step), execute_type=1, return_type=2) if user_song_list[0] and len(user_song_list[1]) != 0: last_user_song_list = [] for user_song in user_song_list[1]: last_user_song_list.append([ user_song[0], user_song[1], int(user_song[3] * config.factor_rank_all_score + user_song[4] * config.factor_rank_week_score + user_song[5] * config.factor_playlist_like_pop + user_song[6] * config.factor_playlist_create_pop + user_song[7] * config.factor_playlist_collect_pop) ]) _database_tool.insert_many_user_song_column( column="score", data_list=last_user_song_list) _database_tool.commit() user_song_count += step logger.debug("parse_user_song_score success", "limit:{}".format(step)) else: break except Exception as e: logger.error("parse_user_song_score fail", "error_type:{},error:{}".format(type(e), e))
def get_neighbors(self, algo: AlgoBase, user_id, k): """ 获取user的近邻列表 :param user_id:用户id :return: """ _database_tool = database_tool() # 内部id user_id_inner = algo.trainset.to_inner_uid(user_id) # 近邻内部id列表 neighbor_id_inner_list = algo.get_neighbors(iid=user_id_inner, k=k) # 近邻外部id列表 neighbor_id_list = (algo.trainset.to_raw_uid(inner_id) for inner_id in neighbor_id_inner_list) # 近邻用户列表 user_neighbors = [] for neighbor_id in neighbor_id_list: user_neighbors.append( _database_tool.select_by_column(table="user", column="user_id", value=neighbor_id)[1]) return user_neighbors
def get_user_ranklist_songs(self, user_id=config.user_id, rank_type=config.rank_type, rank_max=config.rank_max): """ 获取某用户排行榜歌曲列表 :param user_id: 用户id :param rank_max: 排行榜歌曲获取最大数 :return: status: 是否获取到歌曲 :return: 排行榜歌曲列表 """ # 请求参数 _first_param = first_param().get_first_param_ranklist( user_id=user_id, rank_type=rank_type) # 请求数据 ---------------------------------------------------- json_data = None try: content = request_data().get_request_data( first_param=_first_param[1], url=config.url_user_rank) if content[0]: if rank_type == config.rank_type_all: json_data = json.loads(content[1])["allData"] elif rank_type == config.rank_type_week: json_data = json.loads(content[1])["weekData"] else: return False, None except KeyError as e: logger.warning( "get_user_ranklist_songs get failed, Maybe the guy's ranklist is hidden,can you see it in the webpage ?", "user_id:{},rank_type:{},ranklist_url:https://music.163.com/#/user/songs/rank?id={} ,error_type:{},error:{}" .format(user_id, rank_type, user_id, type(e), e)) return False, None except Exception as e: logger.error( "get_user_ranklist_songs get failed", "user_id:{},rank_type:{},error_type:{},error:{}".format( user_id, rank_type, type(e), e)) return False, None # 解析数据 ---------------------------------------------------- ranklist_id = str(user_id) + "r" + str(rank_type) song_success_count = 0 rank_list = [] user_rank_list = [] song_list = [] artist_list = [] artist_song_list = [] song_rank_list = [] user_song_list = [] try: # ranklist_id rank_type rank_date rank_list.append( [ranklist_id, rank_type, int(round(time.time() * 1000))]) # user_id ranklist_id user_rank_list.append([user_id, ranklist_id]) while song_success_count < rank_max and song_success_count < len( json_data): song_id = json_data[song_success_count]["song"]["id"] song_score = json_data[song_success_count]["score"] print(json_data) print('#############################') print(json_data[song_success_count]) print('#############################') print(song_score) # song_id song_name song_list.append( [song_id, json_data[song_success_count]["song"]["name"]]) # song_id ranklist_id song_score song_rank_list.append([song_id, ranklist_id, song_score]) # 多个歌手 artist_count = 0 for artist in json_data[song_success_count]["song"]["ar"]: # artist_id artist_name artist_list.append([artist["id"], artist["name"]]) # artist_id song_id sort artist_song_list.append( [artist["id"], song_id, artist_count]) artist_count += 1 # user_id song_id ranklist_all/week_score user_song_list.append([user_id, song_id, song_score]) song_success_count += 1 except Exception as e: logger.error( "get_user_ranklist_songs parse failed", "user_id:{},rank_type:{},error_type:{},song_success_count:{},error:{}" .format(user_id, rank_type, song_success_count, type(e), e)) return False, None # 存储数据 ---------------------------------------------------- try: _database_tool = database_tool() _database_tool.insert_many_ranklist(rank_list) _database_tool.insert_many_song(song_list) _database_tool.insert_many_artist(artist_list) _database_tool.commit() _database_tool.insert_many_user_ranklist(user_rank_list) _database_tool.insert_many_song_ranklist(song_rank_list) _database_tool.insert_many_artist_song(artist_song_list) _database_tool.insert_many_user_song_column( column="rank_all_score" if rank_type == config.rank_type_all else "rank_week_score", data_list=user_song_list) _database_tool.commit() _database_tool.close() except Exception as e: logger.error( "get_user_ranklist_songs save failed", "user_id:{},rank_type:{},song_count_success,error_type:{},error:{}" .format(user_id, rank_type, song_success_count, type(e), e)) return False, None logger.info( "get_user_ranklist_songs success", "user_id:{},rank_type:{},song_success_count:{}".format( user_id, rank_type, song_success_count)) return True, (rank_list, song_list, artist_list, user_rank_list, song_rank_list, artist_song_list)
data_list=user_song_list) _database_tool.commit() _database_tool.close() except Exception as e: logger.error( "get_user_ranklist_songs save failed", "user_id:{},rank_type:{},song_count_success,error_type:{},error:{}" .format(user_id, rank_type, song_success_count, type(e), e)) return False, None logger.info( "get_user_ranklist_songs success", "user_id:{},rank_type:{},song_success_count:{}".format( user_id, rank_type, song_success_count)) return True, (rank_list, song_list, artist_list, user_rank_list, song_rank_list, artist_song_list) if __name__ == "__main__": _database_tool = database_tool() _database_tool.insert_many_user([[config.user_id, config.user_name]]) _database_tool.commit() _database_tool.close() user_ranklist_songs().get_user_ranklist_songs( user_id=config.user_id, rank_type=config.rank_type_all) user_ranklist_songs().get_user_ranklist_songs( user_id=config.user_id, rank_type=config.rank_type_week) # user_id_list = database_tool().select_user_list(table='user') # user_ranklist_songs().get_user_ranklist_songs_thread(user_list=user_id_list[1], thread_count=5, # thread_inteval_time=2, # rank_max=300)
def get_user_playlists( self, user_id=config.user_id, created_playlists_max=config.created_playlists_max, collected_playlists_max=config.collected_playlists_max, is_playlists_default=config.is_playlists_default, is_playlists_created=config.is_playlists_created, is_playlists_collected=config.is_playlists_collected): """ 获取用户歌单列表 :param user_id: 用户id :param created_playlists_max: 用户创建的歌单获取最大数 :param collected_playlists_max: 用户收集的歌单获取最大数 :param is_playlists_default: 是否爬取”用户喜欢的音乐“歌单 :param is_playlists_created: 是否爬取用户创建的歌单 :param is_playlists_collected: 是否爬取用户收藏的歌单 :return: status: 是否获取到歌单 :return: 歌单列表 :return: 用户歌单列表 """ # 请求参数 ---------------------------------------------------- _first_param = first_param().get_first_param_user_playlists( user_id=user_id) # 请求数据 ---------------------------------------------------- try: content = request_data().get_request_data( _first_param[1], url=config.url_user_playlists) if content[0]: json_playlists_data = json.loads(content[1])["playlist"] else: return False, None except Exception as e: logger.error( "get_user_playlists get failed", "user_id:{},error_type:{},error:{}".format( user_id, type(e), e)) return False, None # 解析数据 ---------------------------------------------------- playlist_count = 0 created_playlists_count = 0 collected_playlists_count = 0 playlist_list = [] user_playlists_list = [] tag_list = [] playlist_tag_list = [] try: while playlist_count < len(json_playlists_data): # ”用户喜欢的音乐“歌单 if is_playlists_default: parse = self.__parser( user_id=user_id, playlist_count=playlist_count, data=json_playlists_data, playlist_type=config.default_playlist) playlist_list.append(parse["playlist_list"]) user_playlists_list.append(parse["user_playlist_list"]) if len(parse["tag_list"]) != 0: tag_list.extend(parse["tag_list"]) playlist_tag_list.extend(parse["playlist_tag_list"]) playlist_count += 1 is_playlists_default = False continue # 用户创建的歌单 elif is_playlists_created: # 若此处出现playlist为0的情况,说明没有爬取“用户喜欢的音乐”歌单,跳过此歌单 if playlist_count == 0: playlist_count += 1 continue # 是否是用户自己创建的歌单 elif str(json_playlists_data[playlist_count]["creator"] ['userId']) == str(user_id): if created_playlists_count < created_playlists_max: parse = self.__parser( user_id=user_id, playlist_count=playlist_count, data=json_playlists_data, playlist_type=config.created_playlist) playlist_list.append(parse["playlist_list"]) user_playlists_list.append( parse["user_playlist_list"]) if len(parse["tag_list"]) != 0: tag_list.extend(parse["tag_list"]) playlist_tag_list.extend( parse["playlist_tag_list"]) created_playlists_count += 1 playlist_count += 1 else: is_playlists_created = False continue # 用户收藏的歌单 elif is_playlists_collected: # 是否是用户收藏的歌单 if str(json_playlists_data[playlist_count]["creator"] ['userId']) != str(user_id): if collected_playlists_count < collected_playlists_max: parse = self.__parser( user_id=user_id, playlist_count=playlist_count, data=json_playlists_data, playlist_type=config.collected_playlist) playlist_list.append(parse["playlist_list"]) user_playlists_list.append( parse["user_playlist_list"]) if len(parse["tag_list"]) != 0: tag_list.extend(parse["tag_list"]) playlist_tag_list.extend( parse["playlist_tag_list"]) collected_playlists_count += 1 # 若此处出现不是用户收藏的歌单情况,说明用户创建歌单没有完全爬取,继续循环 playlist_count += 1 continue break except Exception as e: logger.error( "get_user_playlists parse failed", "user_id:{},created_playlists_count:{},collected_playlists_count:{},error_type:{},error:{}" .format(user_id, created_playlists_count, collected_playlists_count, type(e), e)) return False, None # 存储数据 ---------------------------------------------------- try: _database_tool = database_tool() _database_tool.insert_many_playlist(playlist_list) _database_tool.insert_many_tag(tag_list) _database_tool.commit() _database_tool.insert_many_user_playlist(user_playlists_list) _database_tool.insert_many_playlist_tag(playlist_tag_list) _database_tool.commit() _database_tool.close() except Exception as e: logger.error( "get user_playlist save failed", "user_id:{},created_playlists_count:{},collected_playlists_count:{},error_type:{},error:{}" .format(user_id, created_playlists_count, collected_playlists_count, type(e), e)) return False, None logger.info( "get user_playlist success", "user_id:{},created_playlists_count:{},collected_playlists_count:{}" .format(user_id, created_playlists_count, collected_playlists_count)) return True, playlist_list, user_playlists_list
data[playlist_count]["id"], data[playlist_count]["name"], data[playlist_count]["trackCount"], data[playlist_count]["playCount"], data[playlist_count]["updateTime"] ], "user_playlist_list": [user_id, data[playlist_count]["id"], playlist_type], "tag_list": tag_list, "playlist_tag_list": playlist_tag_list } if __name__ == "__main__": _database_tool = database_tool() _database_tool.insert_many_user([[config.user_id, config.user_name]]) _database_tool.commit() _database_tool.close() max = sys.maxsize # 爬取 喜欢歌单 # print(user_playlists().get_user_playlists(config.user_id, created_playlists_max=max, collected_playlists_max=max, # is_playlists_default=True, is_playlists_created=False, # is_playlists_collected=False)) # # 爬取 创建歌单 # print(user_playlists().get_user_playlists(config.user_id, created_playlists_max=max, collected_playlists_max=max, # is_playlists_default=False, is_playlists_created=True, # is_playlists_collected=False)) # print(user_playlists().get_user_playlists(config.user_id, created_playlists_max=3, collected_playlists_max=max, # is_playlists_default=False, is_playlists_created=True,
def get_playlist_songs_by_playlist_id(self, playlist_id=config.playlist_id, user_id=0, playlist_type=config.playlist_type, playlist_songs_max=config.playlist_songs_max): """ 通过歌单id获取歌单歌曲列表 :param playlist_id: 歌单id :param user_id: 用户id :param playlist_type: 歌单类型,包括 default created collected :param playlist_songs_max: 歌单最大歌曲选取数 :return: status: 是否获取到歌曲 :return: 歌曲个数 :return: 歌曲列表 :return: 歌手列表 :return: 歌曲歌手列表 """ # 请求参数 ---------------------------------------------------- _first_param = first_param().get_first_param_playlist(playlist_id) # 请求数据 ---------------------------------------------------- try: content = request_data().get_request_data(first_param=_first_param[1], url=config.url_playlist) if content[0]: songs_data = json.loads(content[1])["playlist"]["tracks"] tags_data = json.loads(content[1])["playlist"]["tags"] creator_data = json.loads(content[1])["playlist"]["creator"] else: return False, None except Exception as e: logger.error("get_playlist_songs_by_playlist_id get failed", "playlist_id:{},playlist_type:{},error_type:{},error:{}" .format(playlist_id, playlist_type, type(e), e)) return False, None # 解析数据 ---------------------------------------------------- song_count = 0 song_list = [] artist_list = [] artist_song_list = [] song_playlist_list = [] tag_list = tags_data song_tag_list = [] # user_id user_name creator_list = [[ creator_data["userId"], creator_data["nickname"] ]] # user_id playlist_id playlist_type creator_playlist_list = [[ creator_data["userId"], playlist_id, config.created_playlist ]] # 创作者-歌曲列表 creator_song_list = [] user_song_list = [] try: while song_count < playlist_songs_max and song_count < len(songs_data): song_id = songs_data[song_count]["id"] song_pop = int(songs_data[song_count]["pop"]) # song_id song_name song_list.append([ song_id, songs_data[song_count]["name"] ]) # 多个歌手 artist_count = 0 for artist in songs_data[song_count]['ar']: # artist_id artist_name artist_list.append([ artist["id"], artist["name"] ]) # artist_id song_id sort artist_song_list.append([ artist["id"], song_id, artist_count ]) artist_count += 1 # song_id playlist_id song_pop playlist_type song_playlist_list.append([ song_id, playlist_id, song_pop, playlist_type ]) # song_id tag_name for tag in tag_list: song_tag_list.append([ song_id, tag ]) # user_id song_id playlist_like/create/collect_pop user_song_list.append([ user_id, song_id, song_pop ]) # user_id song_id playlist_create_pop creator_song_list.append([ creator_data["userId"], song_id, song_pop ]) song_count += 1 except Exception as e: logger.error("get_playlist_songs_by_playlist_id parse failed", "playlist_id:{},playlist_type:{},song_count:{},error_type:{},error:{}" .format(playlist_id, playlist_type, song_count, type(e), e)) return False, None # 存储数据 ---------------------------------------------------- try: _database_tool = database_tool() _database_tool.insert_many_song(song_list) _database_tool.insert_many_artist(artist_list) _database_tool.insert_many_tag(tag_list) _database_tool.insert_many_user(creator_list) _database_tool.commit() _database_tool.insert_many_song_playlist(song_playlist_list) _database_tool.insert_many_artist_song(artist_song_list) _database_tool.insert_many_song_tag(song_tag_list) _database_tool.insert_many_user_playlist(creator_playlist_list) _database_tool.insert_many_user_song_column(column="playlist_create_pop", data_list=creator_song_list) if playlist_type == config.default_playlist: column = "playlist_like_pop" elif playlist_type == config.created_playlist: column = "playlist_create_pop" elif playlist_type == config.collected_playlist: column = "playlist_collect_pop" else: column = "" if column != "": _database_tool.insert_many_user_song_column(column=column, data_list=user_song_list) _database_tool.commit() _database_tool.close() except Exception as e: logger.error("get_playlist_songs_by_playlist_id save failed", "playlist_id:{},playlist_type:{},error_type:{},error:{}" .format(playlist_id, playlist_type, type(e), e)) return False, None logger.info("get_playlist_songs_by_playlist_id success", "playlist_id:{},playlist_type:{},song_count:{}" .format(playlist_id, playlist_type, song_count)) return True, len(song_list), song_list, artist_list, artist_song_list
def get_song_page_comments(self, song_id, comment_type=config.song_comments_type_hot, offset=0, limit=0, is_get_comment_total=False): """ 通过song_id获取某页评论 :param song_id: 歌曲id :param comment_type: 评论类型,详见config :param offset: 位移量 :param limit: 每页评论限制量 :param is_get_comment_total: 是否为获取评论总数,默认否 :return: 评论数据 """ # 请求参数 ---------------------------------------------------- _first_param = first_param().get_first_param_eapi_comment( offset=offset, limit=limit) if comment_type == config.song_comments_type_hot: url = config.get_comments_hot_url(song_id) eapi_url = config.get_comment_hot_url_for_eapi_param(song_id) elif comment_type == config.song_comments_type_default: url = config.get_comments_default_url(song_id) eapi_url = config.get_comment_default_url_for_eapi_param(song_id) else: return False, None # 请求数据 ---------------------------------------------------- try: content = request_data().get_request_data( first_param=_first_param[1], url=url, api_type=config.api_eapi, eapi_url=eapi_url) if content[0]: if is_get_comment_total: return True, json.loads(content[1])["total"] elif comment_type == config.song_comments_type_default: json_data = json.loads(content[1])["comments"] elif comment_type == config.song_comments_type_hot: json_data = json.loads(content[1])["hotComments"] else: return False, None else: return False, None except Exception as e: logger.error( "get_song_page_comments get failed", "song_id:{},comment_type:{},offset:{},limit:{},error_type:{},error:{}" .format(song_id, comment_type, offset, limit, type(e), e)) return False, [] # 解析数据 ---------------------------------------------------- user_list = [] comment_list = [] user_comment_list = [] song_comment_list = [] try: for comment_json in json_data: # user_id user_name user_list.append([ comment_json["user"]["userId"], comment_json["user"]["nickname"] ]) # comment_id comment_type comment_date comment_content comment_like_count comment_list.append([ comment_json["commentId"], comment_type, comment_json["time"], comment_json["content"], comment_json["likedCount"] ]) # user_id comment_id user_comment_list.append([ comment_json["user"]["userId"], comment_json["commentId"] ]) # song_id comment_id song_comment_list.append([song_id, comment_json["commentId"]]) # logger.info("get_song_page_comments parse success", # "song_id:{},comment_total:{},comment_type:{},offset:{},limit:{}" # .format(song_id, len(comment_list), comment_type, offset, limit)) except Exception as e: logger.error( "get_song_page_comments parse failed", "song_id:{},comment_type:{},offset:{},limit:{},error_type:{},error:{}" .format(song_id, comment_type, offset, limit, type(e), e)) # 存储数据 ---------------------------------------------------- try: _database_tool = database_tool() _database_tool.insert_many_user(user_list) _database_tool.insert_many_comment(comment_list) _database_tool.commit() _database_tool.insert_many_song_comment(song_comment_list) _database_tool.insert_many_user_comment(user_comment_list) _database_tool.commit() _database_tool.close() except Exception as e: logger.error( "get_song_page_comments save failed", "song_id:{},comment_type:{},offset:{},limit:{},error_type:{},error:{}" .format(song_id, comment_type, offset, limit, type(e), e)) return True, (user_list, comment_list, user_comment_list)