def parse_load(self, response):
request_url = response.request.url
p = re.compile('&pre_page=(\d).*&page=(\d)') #用于判断是第一页的第一次加载
match = p.search(request_url)
if match:
if int(match.group(1)) == 0 and int(
match.group(2)) == 1: #进行当前主用户信息的获取(即非@用户和转发用户)
is_search = response.meta['is_search']
if not is_search: #没有搜索过该主用户
analyzer = Analyzer()
total_pq = analyzer.get_html(
response.body, 'script:contains("PCD_person_info")')
user_property = analyzer.get_userproperty(total_pq)
if not user_property == 'icon_verify_co_v': #该账号不为公众账号
userinfo_url = analyzer.get_userinfohref(total_pq)
yield Request(url=userinfo_url,
meta={
'cookiejar':
response.meta['cookiejar'],
'uid': response.meta['uid'],
'is_friend': 0
},
callback=self.parse_userinfo)
item = WeibospiderItem() #获取用户微博信息及@用户信息
analyzer = Analyzer()
friendcircle = FriendCircle()
total_pq = analyzer.get_html(response.body,
'script:contains("WB_feed WB_feed_v3")')
item['uid'] = response.meta['uid']
item['content'] = analyzer.get_content(total_pq)
item['time'], item['timestamp'] = analyzer.get_time(total_pq)
atuser_info, item['repost_user'] = analyzer.get_atuser_repostuser(
total_pq)
atuser_list = friendcircle.atuser_parser(atuser_info)
item['atuser_nickname_list'] = atuser_list
yield item
frc_analyzer = friendcircle_analyzer()
#获取@用户uid及基本信息
atuser_set = self.get_atuser_set(atuser_list)
# for atuser_alias in atuser_set:
# friend_url = frc_analyzer.get_frienduid_url(atuser_alias)
# yield Request(url=friend_url,meta={'cookiejar':response.meta['cookiejar'],'uid':response.meta['uid'],'is_friend':1},callback=self.parse_friend_uid) #is_friend=1代表爬取@用户基本信息
#获取转发用户uid及基本信息
for repostuser_alias in item['repost_user']:
if repostuser_alias: #repostuser_alias不为空，即有转发用户
friend_url = frc_analyzer.get_frienduid_url(repostuser_alias)
yield Request(
url=friend_url,
meta={
'cookiejar': response.meta['cookiejar'],
'uid': response.meta['uid'],
'is_friend': 2
},
callback=self.parse_friend_uid) #is_friend=2代表爬取转发用户基本信息
def parse_friend_uid(self,response):
'''根据昵称解析朋友圈用户uid'''
frc_analyzer = friendcircle_analyzer()
friend_json_info = frc_analyzer.get_frienduid(response.body)
if friend_json_info and friend_json_info['code'] != 100001:
if friend_json_info['uid'] and friend_json_info['tag'] == 0: #获取朋友圈非公众账号个人信息
print "the friend uid:",friend_json_info['uid']
userinfo_url = frc_analyzer.get_userinfo_url(friend_json_info['uid'])
yield Request(url=userinfo_url,meta={'cookiejar':response.meta['cookiejar'],'uid':friend_json_info['uid'],'is_friend':response.meta['is_friend']},callback=self.parse_userinfo,dont_filter=True)
else:
logger.warning('no right uid found for alias!')
else:
logger.warning('no right uid found for alias!')
def parse_load(self,response):
request_url = response.request.url
p=re.compile('&pre_page=(\d).*&page=(\d)') #用于判断是第一页的第一次加载
match = p.search(request_url)
if match:
if int(match.group(1)) == 0 and int(match.group(2)) == 1: #进行当前主用户信息的获取(即非@用户和转发用户)
is_search = response.meta['is_search']
if not is_search: #没有搜索过该主用户，则is_search=0
analyzer = Analyzer()
total_pq = analyzer.get_html(response.body,'script:contains("PCD_person_info")')
user_property = analyzer.get_userproperty(total_pq)
if not user_property == 'icon_verify_co_v': #该账号不为公众账号
userinfo_url = analyzer.get_userinfohref(total_pq)
yield Request(url=userinfo_url,cookies=random.choice(COOKIES),meta={'uid':response.meta['uid'],'is_friend':0},callback=self.parse_userinfo)
item = WeibospiderItem() #获取用户微博信息及@用户与转发信息
analyzer = Analyzer()
friendcircle = FriendCircle()
total_pq = analyzer.get_html(response.body,'script:contains("WB_feed WB_feed_v3")')
item['uid'] = response.meta['uid']
item['content'] = analyzer.get_content(total_pq)
item['time'],item['timestamp'] = analyzer.get_time(total_pq)
weibo_analyzer = weibocontent_analyzer()
item['repost_nums'],item['comment_nums'],item['like_nums'] = weibo_analyzer.get_weibo_relative_args(total_pq)
atuser_info,item['repost_user'] = analyzer.get_atuser_repostuser(total_pq)
atuser_list = friendcircle.atuser_parser(atuser_info)
item['atuser_nickname_list'] = atuser_list
yield item
frc_analyzer = friendcircle_analyzer()
#获取@用户uid及基本信息
atuser_set = self.get_atuser_set(atuser_list)
for atuser_alias in atuser_set:
friend_url = frc_analyzer.get_frienduid_url(atuser_alias)
yield Request(url=friend_url,cookies=random.choice(COOKIES),meta={'uid':response.meta['uid'],'is_friend':1},callback=self.parse_friend_uid) #is_friend=1代表爬取@用户基本信息
#获取转发用户uid及基本信息
for repostuser_alias in item['repost_user']:
if repostuser_alias: #repostuser_alias不为空，即有转发用户
friend_url = frc_analyzer.get_frienduid_url(repostuser_alias)
yield Request(url=friend_url,cookies=random.choice(COOKIES),meta={'uid':response.meta['uid'],'is_friend':2},callback=self.parse_friend_uid) #is_friend=2代表爬取转发用户基本信息
def parse_friend_uid(self, response):
'''根据昵称解析朋友圈用户uid'''
frc_analyzer = friendcircle_analyzer()
friend_json_info = frc_analyzer.get_frienduid(response.body)
if friend_json_info and friend_json_info['code'] != 100001:
if friend_json_info['uid'] and friend_json_info[
'tag'] == 0: #获取朋友圈非公众账号个人信息
print "the friend uid:", friend_json_info['uid']
userinfo_url = frc_analyzer.get_userinfo_url(
friend_json_info['uid'])
yield Request(url=userinfo_url,
cookies=random.choice(COOKIES),
meta={
'uid': friend_json_info['uid'],
'is_friend': response.meta['is_friend']
},
callback=self.parse_userinfo,
dont_filter=True)
else:
logger.warning('no right uid found for alias!')
else:
logger.warning('no right uid found for alias!')