def parse_show_more_more_replies(self, response):
"""每个二级评论块可能有--另外？条回复"""
try:
res = json.loads(response.text)
except:
return []
comment_list3_response = HtmlResponse(url=response.url, body=res.get('conversation_html'), encoding='utf-8')
comment_list3_response.meta2 = copy.deepcopy(response.meta)
comments = self.parse_secondary_comment(comment_list3_response)
for comment in comments: # 创建评论对象，同时更新社交关系
if isinstance(comment, scrapy.Request):
yield comment
else:
comment.pop("is_end_comment")
# 被评论人等信息
data_reply_to_users = comment.pop("data_reply_to_users")
new_relations_dct = comment.pop("NEW_RELATIONS")
comment_item = CommentItem(**comment) # pop后就创建字典
yield comment_item
# social_relations = self.make_social_relation_dict(data_reply_to_users, comment)
# for relation in social_relations:
# relation_item = SocialRelationItem(ListSocialRelation=[])
# relation_item["ListSocialRelation"].append(relation)
# yield relation_item
social_relation = self.make_social_relation_dict_bad(new_relations_dct, comment)
relation_item = SocialRelationItem(**social_relation)
yield relation_item
def parse_other_comment_page(self, response):
"""解析其他页一级评论页(json格式)"""
# 翻页做准备
commentator_account = response.meta.get("commentator_account")
comment_id = response.meta.get("comment_id")
# 评论翻页计数
current_comment_page = response.meta.get("current_comment_page")
is_new = response.meta.get("is_new")
# if self.increment_crawl and not is_new and current_comment_page >= 3:
# return []
try:
res = json.loads(response.text)
except:
return []
comment_list_response = HtmlResponse(url=response.url,
body=res.get('items_html'),
encoding='utf-8')
comment_list_response.meta2 = copy.deepcopy(response.meta)
comments = self.parse_comment(comment_list_response)
for comment in comments: # 创建评论对象，同时更新社交关系
if isinstance(comment, scrapy.Request):
yield comment
else:
if comment.pop("is_end_comment"): # 增量爬取，终止继续采集评论
return []
# 被评论人等信息
data_reply_to_users = comment.pop("data_reply_to_users")
new_relations_dct = comment.pop("NEW_RELATIONS")
comment_item = CommentItem(**comment) # pop后就创建字典
yield comment_item
# social_relations = self.make_social_relation_dict(data_reply_to_users, comment)
# for relation in social_relations:
# relation_item = SocialRelationItem(ListSocialRelation=[])
# relation_item["ListSocialRelation"].append(relation)
# yield relation_item
social_relation = self.make_social_relation_dict_bad(
new_relations_dct, comment)
relation_item = SocialRelationItem(**social_relation)
yield relation_item
# 其他一级评论页 再翻页
has_more_comment = self.has_more_comment(comment_list_response)
has_next = res.get('min_position') or has_more_comment
if has_next: # 是否含有下一页
next_url = 'https://twitter.com/i/' + commentator_account + '/conversation/' + comment_id + '?include_available_features=1&include_entities=1&max_position=' + has_next
if len(next_url) >= 10000:
return []
else:
meta = copy.deepcopy(response.meta)
meta["current_comment_page"] = current_comment_page + 1
yield scrapy.Request(url=next_url,
callback=self.parse_other_comment_page,
meta=meta)
def parse_secondary_comment_page(self, response):
"""解析回复数大于1的评论首页 Html格式
得到二级评论
"""
# 翻页做准备
commentator_account = response.meta.get("commentator_account")
comment_id = response.meta.get("comment_id")
current_secondary_comment_page = 1
response.meta2 = copy.deepcopy(response.meta)
response.meta2[
"current_secondary_comment_page"] = current_secondary_comment_page
comments = self.parse_secondary_comment(response) # 解析二级评论
for comment in comments: # 创建评论对象，同时更新社交关系
if isinstance(comment, scrapy.Request):
yield comment
else:
if comment.pop("is_end_comment"):
return []
# 被评论人等信息
data_reply_to_users = comment.pop("data_reply_to_users")
new_relations_dct = comment.pop("NEW_RELATIONS")
comment_item = CommentItem(**comment) # pop后就创建字典
yield comment_item
# social_relations = self.make_social_relation_dict(data_reply_to_users, comment)
# for relation in social_relations:
# relation_item = SocialRelationItem(ListSocialRelation=[])
# relation_item["ListSocialRelation"].append(relation)
# yield relation_item
social_relation = self.make_social_relation_dict_bad(
new_relations_dct, comment)
relation_item = SocialRelationItem(**social_relation)
yield relation_item
"""二级评论首页翻页 """
# 下一页评论地址(第二页)
xpath_next_page = "//div[@class='ThreadedDescendants']/div[contains(@class,'stream-container')]/@data-min-position"
has_next_page = response.xpath(xpath_next_page).extract_first()
# 显示更多回复
xpath_more_comment = "//li[@class='ThreadedConversation-showMoreThreads']/button/@data-cursor"
has_more_comment = response.xpath(xpath_more_comment).extract_first()
has_next = has_next_page or has_more_comment
if has_next: # 情况1,还有评论就继续传递更新社交关系, 没有更多二级评论就不用管
next_url = 'https://twitter.com/i/' + commentator_account + '/conversation/' + comment_id + '?include_available_features=1&include_entities=1&max_position=' + has_next
meta = copy.deepcopy(response.meta)
meta[
"current_secondary_comment_page"] = current_secondary_comment_page + 1
yield scrapy.Request(
url=next_url,
callback=self.parse_other_secondary_comment_page,
meta=meta)
def parse_article_detail(self, response):
"""解析推特帖子详情页 以及首页评论"""
current_page = response.meta.get("current_page")
article_count = response.meta.get("article_count")
is_new = response.meta.get("is_new")
user_id = response.meta.get("user_id")
user_account = response.meta.get("UserAccount") # 帖子作者(不一定是帖子原作者，而是所查询的用户)
user_nick = response.meta.get("AuthorNick") # 帖子作者昵称(不一定是帖子原作者，而是所查询的用户)
XXX_UserAccount = response.meta.get("XXX_UserAccount")
XXX_AuthorNick = response.meta.get("XXX_AuthorNick")
# 发布时间
xpath_publish_time = "//div[contains(@class,'permalink-inner permalink-tweet-container')]//small[@class='time']//span[@data-time]/@data-time"
ptime = response.xpath(xpath_publish_time).extract_first()
if not ptime:
return []
p_time = int(ptime.strip())
publish_time = str(datetime.datetime.fromtimestamp(p_time))
if current_page > 1: # 每个账号必采1页
if not self.increment_crawl or is_new: # 首次采集/新增账号
if p_time < self.e_time and article_count["count"] >= self.article_count_limit:
article_count["end_collect"] = True
return []
else: # 增量爬取/且不是新增账号，大于第1页的采(self.inc_e_time)天以内
if p_time < self.inc_e_time:
article_count["end_collect"] = True
return []
# 继续采集 +1
article_count["count"] += 1
# 帖子url
article_url = response.url
# 文章ID
# article_id = url.split('/')[-1]
xpath_article_id = "//div[contains(@class,'permalink-inner permalink-tweet-container')]//div[contains(@class,'js-actionable-user')]/@data-item-id"
article_id = response.xpath(xpath_article_id).extract_first()
# (原作者)作者账号 (作用：1.下一页拼接 2.社交关系)
xpath_origin_author = "//div[contains(@class,'permalink-inner permalink-tweet-container')]//div[contains(@class,'js-actionable-user')]/@data-screen-name"
origin_author = response.xpath(xpath_origin_author).extract_first()
# (原作者)作者昵称
xpath_origin_author_nick = "//div[contains(@class,'permalink-inner permalink-tweet-container')]//div[contains(@class,'js-actionable-user')]/@data-name"
origin_author_nickname = response.xpath(xpath_origin_author_nick).extract_first()
# (原作者)作者ID
xpath_origin_author_id = "//div[contains(@class,'permalink-inner permalink-tweet-container')]//div[contains(@class,'js-actionable-user')]/@data-user-id"
origin_author_id = response.xpath(xpath_origin_author_id).extract_first()
# 帖子内容(1.文本内容 2.视频内容 3.(未采集)其他底部视频内容(包含时间、转推、点赞等)[js-tweet-details-fixer tweet-details-fixer])
xpath_content = "//div[contains(@class,'permalink-inner permalink-tweet-container')]//div[@class='js-tweet-text-container'] | //div[contains(@class,'permalink-inner permalink-tweet-container')]//div[@class='AdaptiveMediaOuterContainer'] | //div[contains(@class,'permalink-inner permalink-tweet-container')]//div[contains(@class,'u-block js-tweet-details-fixer')]"
content = ''.join(response.xpath(xpath_content).extract())
# 帖子内容语言代码
xpath_content_language_code = "//div[contains(@class,'permalink-inner permalink-tweet-container')]//p/@lang"
content_language_code = response.xpath(xpath_content_language_code).extract_first()
# 评论数
xpath_comment = "//div[contains(@class,'permalink-inner permalink-tweet-container')]//div[contains(@class,'stream-item-footer')]//div[contains(@class,'ProfileTweet-actionCountList')]//span[contains(@class,'ProfileTweet-action--reply')]/span[@class='ProfileTweet-actionCount']/@data-tweet-stat-count"
comments = response.xpath(xpath_comment).extract_first()
comments_count = int(comments.strip()) if comments else 0
# 转推数
xpath_retweets = "//div[contains(@class,'permalink-inner permalink-tweet-container')]//div[contains(@class,'stream-item-footer')]//div[contains(@class,'ProfileTweet-actionCountList')]//span[contains(@class,'ProfileTweet-action--retweet')]/span[@class='ProfileTweet-actionCount']/@data-tweet-stat-count"
retweets = response.xpath(xpath_retweets).extract_first()
retweets_count = int(retweets) if retweets else 0
# 点赞数
xpath_likes = "//div[contains(@class,'permalink-inner permalink-tweet-container')]//div[contains(@class,'stream-item-footer')]//div[contains(@class,'ProfileTweet-actionCountList')]//span[contains(@class,'ProfileTweet-action--favorite')]/span[@class='ProfileTweet-actionCount']/@data-tweet-stat-count"
likes = response.xpath(xpath_likes).extract_first()
likes_count = int(likes) if likes else 0
# 帖子内容下面部分的iframe标签里面的html
xpath_additional_url = "//div[contains(@class,'permalink-inner permalink-tweet-container')]//div[contains(@class,'js-macaw-cards-iframe-container')]/@data-full-card-iframe-url"
additional_url = response.xpath(xpath_additional_url).extract_first()
additional_content = self.parse_related_article(url=response.urljoin(additional_url))
content = content + additional_content if additional_content else content
# 视频地址
video_url = response.xpath("//meta[@property='og:video:secure_url']/@content").extract_first()
if not content: # 丢弃没有内容的文章
return
if self.user_msg_choose == 1: # 假
user_id_choose = user_id
account = user_account
user_account_choose = XXX_UserAccount
user_nick_choose = XXX_AuthorNick
elif self.user_msg_choose == 2: # 伪
user_id_choose = user_id
account = user_account
user_account_choose = user_account
user_nick_choose = user_nick
else: # 真
user_id_choose = origin_author_id
account = origin_author
user_account_choose = origin_author
user_nick_choose = origin_author_nickname
article_data = {
"PNID": article_id,
"Url": article_url,
"Author": user_account_choose, # 传过来的账号
"AuthorNick": user_nick_choose, # 传过来的昵称
"PublishTime": publish_time,
"Content": content,
"CommentCount": comments_count, # 评论数
"ForwardNum": retweets_count, # 转载数
"ClickCount": likes_count, # 点击数(点赞数？)
"LanguageCode": content_language_code or 'en', # 语言编码 response.meta.get("LanguageCode")
"Title": "",
"Abstract": "",
"Keywords": "",
"VideoUrl": video_url,
"MediaSourceUrl": video_url or "",
"is_new": is_new,
"user_id": user_id_choose,
"account": account
}
article_item = ArticleItem(**article_data)
yield article_item
"""解析帖子页-->首页评论"""
current_comment_page = 1
response.meta2 = copy.deepcopy(response.meta)
response.meta2['article_id'] = article_id
response.meta2['article_url'] = article_url
# 假/伪/真
response.meta2['user_id_choose'] = user_id_choose
response.meta2['user_account_choose'] = user_account_choose
response.meta2['user_nick_choose'] = user_nick_choose
response.meta2['account'] = account
response.meta2['current_comment_page'] = current_comment_page # 一级评论的页数
"""解析评论"""
comments = self.parse_comment(response)
for comment in comments: # 创建评论对象，同时更新社交关系
if isinstance(comment, scrapy.Request):
yield comment
else:
if comment.pop("is_end_comment"):
return []
# 被评论人等信息
data_reply_to_users = comment.pop("data_reply_to_users")
new_relations_dct = comment.pop("NEW_RELATIONS")
comment_item = CommentItem(**comment) # pop后就创建字典
yield comment_item
# social_relations = self.make_social_relation_dict(data_reply_to_users, comment)
# for relation in social_relations:
# relation_item = SocialRelationItem(ListSocialRelation=[])
# relation_item["ListSocialRelation"].append(relation)
# yield relation_item
social_relation = self.make_social_relation_dict_bad(new_relations_dct, comment)
relation_item = SocialRelationItem(**social_relation)
yield relation_item
# 下一页评论地址(第二页)
xpath_next_page = "//div[@class='ThreadedDescendants']/div[contains(@class,'stream-container')]/@data-min-position"
has_next_page = response.xpath(xpath_next_page).extract_first()
# 显示更多回复
xpath_more_comment = "//li[@class='ThreadedConversation-showMoreThreads']/button/@data-cursor"
has_more_comment = response.xpath(xpath_more_comment).extract_first()
has_next = has_next_page or has_more_comment
if has_next:
next_url = 'https://twitter.com/i/' + origin_author + '/conversation/' + article_id + '?include_available_features=1&include_entities=1&max_position=' + has_next
meta = copy.deepcopy(response.meta2)
meta["commentator_account"] = origin_author
meta["comment_id"] = article_id
meta["current_comment_page"] = current_comment_page + 1 # 第二页的评论
yield scrapy.Request(url=next_url, callback=self.parse_other_comment_page, meta=meta)