def parse_total_page(self, response):
analyzer = Analyzer()
total_pq = analyzer.get_html(response.body,
'script:contains("W_pages")')
friendcircle_analyzer = keyword_info_analyzer()
total_pages = friendcircle_analyzer.get_totalpages(
total_pq) #需要爬取的微博朋友圈页数
logger.info("the total_pages is: %d", total_pages)
getweibopage = GetWeibopage()
mainpage_url = response.meta['mainpage_url']
user_id = response.meta['uid']
is_search = response.meta['is_search']
for page in range(1): #TODO 此处要更改为total_pages
GetWeibopage.data['uid'] = user_id
GetWeibopage.data['page'] = page + 1
firstload_url = mainpage_url + getweibopage.get_firstloadurl()
yield Request(url=firstload_url,
meta={
'cookiejar': response.meta['cookiejar'],
'uid': user_id,
'is_search': is_search
},
callback=self.parse_load)
secondload_url = mainpage_url + getweibopage.get_secondloadurl()
#yield Request(url=secondload_url,meta={'cookiejar':response.meta['cookiejar'],'uid':user_id,'is_search':is_search},callback=self.parse_load)
thirdload_url = mainpage_url + getweibopage.get_thirdloadurl()
def parse_total_page(self,response):
analyzer = Analyzer()
total_pq = analyzer.get_html(response.body,'script:contains("W_pages")')
friendcircle_analyzer = keyword_info_analyzer()
total_pages = friendcircle_analyzer.get_totalpages(total_pq) #需要爬取的微博朋友圈页数
logger.info("the total_pages is: %d",total_pages)
getweibopage = GetWeibopage()
mainpage_url = response.meta['mainpage_url']
user_id = response.meta['uid']
is_search = response.meta['is_search']
for page in range(1): #TODO 此处要更改为total_pages
GetWeibopage.data['uid'] = user_id
GetWeibopage.data['page'] = page + 1
firstload_url = mainpage_url + getweibopage.get_firstloadurl()
yield Request(url=firstload_url,meta={'cookiejar':response.meta['cookiejar'],'uid':user_id,'is_search':is_search},callback=self.parse_load)
secondload_url = mainpage_url + getweibopage.get_secondloadurl()
#yield Request(url=secondload_url,meta={'cookiejar':response.meta['cookiejar'],'uid':user_id,'is_search':is_search},callback=self.parse_load)
thirdload_url = mainpage_url + getweibopage.get_thirdloadurl()