def request_comments(ctoken, replies=False): if replies: # let's make it use different urls for no reason despite all the data being encoded base_url = "https://m.youtube.com/watch_comment?action_get_comment_replies=1&ctoken=" else: base_url = "https://m.youtube.com/watch_comment?action_get_comments=1&ctoken=" url = base_url + ctoken.replace("=", "%3D") + "&pbj=1" for i in range(0, 8): # don't retry more than 8 times content = util.fetch_url(url, headers=mobile_headers, report_text="Retrieved comments", debug_name='request_comments') if content[ 0: 4] == b")]}'": # random closing characters included at beginning of response for some reason content = content[4:] elif content[ 0: 10] == b'\n<!DOCTYPE': # occasionally returns html instead of json for no reason content = b'' print("got <!DOCTYPE>, retrying") continue break polymer_json = json.loads(util.uppercase_escape(content.decode('utf-8'))) return polymer_json
def playlist_first_page(playlist_id, report_text="Retrieved playlist"): url = 'https://m.youtube.com/playlist?list=' + playlist_id + '&pbj=1' content = util.fetch_url(url, util.mobile_ua + headers_1, report_text=report_text, debug_name='playlist_first_page') content = json.loads(util.uppercase_escape(content.decode('utf-8'))) return content
def playlist_first_page(playlist_id, report_text="Retrieved playlist"): url = 'https://m.youtube.com/playlist?list=' + playlist_id + '&pbj=1' content = util.fetch_url(url, util.mobile_ua + headers_1, report_text=report_text) '''with open('debug/playlist_debug', 'wb') as f: f.write(content)''' content = json.loads(util.uppercase_escape(content.decode('utf-8'))) return content
def get_videos(playlist_id, page): url = "https://m.youtube.com/playlist?ctoken=" + playlist_ctoken(playlist_id, (int(page)-1)*20) + "&pbj=1" headers = { 'User-Agent': ' Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1', 'Accept': '*/*', 'Accept-Language': 'en-US,en;q=0.5', 'X-YouTube-Client-Name': '2', 'X-YouTube-Client-Version': '2.20180508', } content = util.fetch_url(url, headers, report_text="Retrieved playlist", debug_name='playlist_videos') info = json.loads(util.uppercase_escape(content.decode('utf-8'))) return info
def parse_comments_ajax(content, replies=False): try: content = json.loads(util.uppercase_escape(content.decode('utf-8'))) #print(content) comments_raw = content['content']['continuation_contents']['contents'] ctoken = util.default_multi_get(content, 'content', 'continuation_contents', 'continuations', 0, 'continuation', default='') comments = [] for comment_raw in comments_raw: replies_url = '' if not replies: if comment_raw['replies'] is not None: reply_ctoken = comment_raw['replies']['continuations'][0][ 'continuation'] comment_id, video_id = get_ids(reply_ctoken) replies_url = util.URL_ORIGIN + '/comments?parent_id=' + comment_id + "&video_id=" + video_id comment_raw = comment_raw['comment'] comment = { 'author': comment_raw['author']['runs'][0]['text'], 'author_url': comment_raw['author_endpoint']['url'], 'author_channel_id': '', 'author_id': '', 'author_avatar': comment_raw['author_thumbnail']['url'], 'likes': comment_raw['like_count'], 'published': comment_raw['published_time']['runs'][0]['text'], 'text': comment_raw['content']['runs'], 'reply_count': '', 'replies_url': replies_url, } comments.append(comment) except Exception as e: print('Error parsing comments: ' + str(e)) comments = () ctoken = '' return {'ctoken': ctoken, 'comments': comments}
def parse_comments_polymer(content, replies=False): try: video_title = '' content = json.loads(util.uppercase_escape(content.decode('utf-8'))) url = content[1]['url'] ctoken = urllib.parse.parse_qs(url[url.find('?') + 1:])['ctoken'][0] video_id = ctoken_metadata(ctoken)['video_id'] #print(content) try: comments_raw = content[1]['response']['continuationContents'][ 'commentSectionContinuation']['items'] except KeyError: comments_raw = content[1]['response']['continuationContents'][ 'commentRepliesContinuation']['contents'] replies = True ctoken = util.default_multi_get(content, 1, 'response', 'continuationContents', 'commentSectionContinuation', 'continuations', 0, 'nextContinuationData', 'continuation', default='') comments = [] for comment_raw in comments_raw: replies_url = '' view_replies_text = '' try: comment_raw = comment_raw['commentThreadRenderer'] except KeyError: pass else: if 'commentTargetTitle' in comment_raw: video_title = comment_raw['commentTargetTitle']['runs'][0][ 'text'] parent_id = comment_raw['comment']['commentRenderer'][ 'commentId'] # TODO: move this stuff into the comments_html function if 'replies' in comment_raw: #reply_ctoken = comment_raw['replies']['commentRepliesRenderer']['continuations'][0]['nextContinuationData']['continuation'] #comment_id, video_id = get_ids(reply_ctoken) replies_url = util.URL_ORIGIN + '/comments?parent_id=' + parent_id + "&video_id=" + video_id view_replies_text = yt_data_extract.get_plain_text( comment_raw['replies']['commentRepliesRenderer'] ['moreText']) match = reply_count_regex.search(view_replies_text) if match is None: view_replies_text = '1 reply' else: view_replies_text = match.group(1) + " replies" elif not replies: view_replies_text = "Reply" replies_url = util.URL_ORIGIN + '/post_comment?parent_id=' + parent_id + "&video_id=" + video_id comment_raw = comment_raw['comment'] comment_raw = comment_raw['commentRenderer'] comment = { 'author_id': comment_raw.get('authorId', ''), 'author_avatar': comment_raw['authorThumbnail']['thumbnails'][0]['url'], 'likes': comment_raw['likeCount'], 'published': yt_data_extract.get_plain_text( comment_raw['publishedTimeText']), 'text': comment_raw['contentText'].get('runs', ''), 'view_replies_text': view_replies_text, 'replies_url': replies_url, 'video_id': video_id, 'comment_id': comment_raw['commentId'], } if 'authorText' in comment_raw: # deleted channels have no name or channel link comment['author'] = yt_data_extract.get_plain_text( comment_raw['authorText']) comment['author_url'] = comment_raw['authorEndpoint'][ 'commandMetadata']['webCommandMetadata']['url'] comment['author_channel_id'] = comment_raw['authorEndpoint'][ 'browseEndpoint']['browseId'] else: comment['author'] = '' comment['author_url'] = '' comment['author_channel_id'] = '' comments.append(comment) except Exception as e: print('Error parsing comments: ' + str(e)) comments = () ctoken = '' return {'ctoken': ctoken, 'comments': comments, 'video_title': video_title}
def parse_comments_polymer(content): try: video_title = '' content = json.loads(util.uppercase_escape(content.decode('utf-8'))) url = content[1]['url'] ctoken = urllib.parse.parse_qs(url[url.find('?')+1:])['ctoken'][0] metadata = ctoken_metadata(ctoken) try: comments_raw = content[1]['response']['continuationContents']['commentSectionContinuation']['items'] except KeyError: comments_raw = content[1]['response']['continuationContents']['commentRepliesContinuation']['contents'] ctoken = util.default_multi_get(content, 1, 'response', 'continuationContents', 'commentSectionContinuation', 'continuations', 0, 'nextContinuationData', 'continuation', default='') comments = [] for comment_json in comments_raw: number_of_replies = 0 try: comment_thread = comment_json['commentThreadRenderer'] except KeyError: comment_renderer = comment_json['commentRenderer'] else: if 'commentTargetTitle' in comment_thread: video_title = comment_thread['commentTargetTitle']['runs'][0]['text'] if 'replies' in comment_thread: view_replies_text = yt_data_extract.get_plain_text(comment_thread['replies']['commentRepliesRenderer']['moreText']) view_replies_text = view_replies_text.replace(',', '') match = re.search(r'(\d+)', view_replies_text) if match is None: number_of_replies = 1 else: number_of_replies = int(match.group(1)) comment_renderer = comment_thread['comment']['commentRenderer'] comment = { 'author_id': comment_renderer.get('authorId', ''), 'author_avatar': comment_renderer['authorThumbnail']['thumbnails'][0]['url'], 'likes': comment_renderer['likeCount'], 'published': yt_data_extract.get_plain_text(comment_renderer['publishedTimeText']), 'text': comment_renderer['contentText'].get('runs', ''), 'number_of_replies': number_of_replies, 'comment_id': comment_renderer['commentId'], } if 'authorText' in comment_renderer: # deleted channels have no name or channel link comment['author'] = yt_data_extract.get_plain_text(comment_renderer['authorText']) comment['author_url'] = comment_renderer['authorEndpoint']['commandMetadata']['webCommandMetadata']['url'] comment['author_channel_id'] = comment_renderer['authorEndpoint']['browseEndpoint']['browseId'] else: comment['author'] = '' comment['author_url'] = '' comment['author_channel_id'] = '' comments.append(comment) except Exception as e: print('Error parsing comments: ' + str(e)) comments = () ctoken = '' return { 'ctoken': ctoken, 'comments': comments, 'video_title': video_title, 'video_id': metadata['video_id'], 'offset': metadata['offset'], 'is_replies': metadata['is_replies'], 'sort': metadata['sort'], }