def parseVideoPageJson(self, response): if response.status != 200: print('get url error: ' + response.url) return account = response.meta['account'] rltJson = json.loads(response.text) contentList = rltJson['data'] curTime = dateUtil.getCurDate() for contentInfo in contentList: contentItem = ContentItem() contentItem['channel_id'] = self.channel_id contentItem['account_id'] = account contentItem['record_class'] = "content_info" contentItem['crawl_time'] = curTime contentItem['id'] = contentInfo['id'] contentItem['title'] = contentInfo['title'] contentItem['content_link'] = contentInfo['playLink'] contentItem['publish_time'] = time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime((int(contentInfo['createTime'])) / 1000)) contentItem['read_count'] = contentInfo['playCount'] contentItem['comment_count'] = contentInfo['commentCount'] contentItem['like_count'] = contentInfo['likeCount'] yield contentItem
def parseArticlePageJson(self, response): if response.status != 200: print('get url error: ' + response.url) return account = response.meta['account'] cookie = response.meta['cookie'] currentPage = response.meta['currentPage'] totalPage = response.meta['totalPage'] beginFlag = response.meta['beginFlag'] rltJson = json.loads(response.text) if beginFlag: total = int(rltJson['total']) totalPage = math.ceil(total / 10) beginFlag = False contentList = rltJson['list'] curTime = dateUtil.getCurDate() for contentInfo in contentList: contentItem = ContentItem() contentItem['channel_id'] = self.channel_id contentItem['account_id'] = account contentItem['record_class'] = "content_info" contentItem['crawl_time'] = curTime contentItem['id'] = contentInfo['id'] contentItem['title'] = contentInfo['title'] contentItem['content_link'] = contentInfo['url'] contentItem['publish_time'] = time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(int(contentInfo['updatedAt']))) contentItem['read_count'] = contentInfo['readingNum'] contentItem['comment_count'] = contentInfo['commentsNum'] contentItem['share_count'] = contentInfo['forwardingNum'] contentItem['collect_count'] = contentInfo['collectionNum'] contentItem['recommend_count'] = contentInfo['recommendedNum'] status = int(contentInfo['status']) # 搜狗:1-已发布;40-未通过;134-草稿 contentItem['publish_status'] = publicContentStatus[ channelContentStatus[self.channel_id][status]] yield contentItem currentPage += 1 if currentPage <= totalPage: time.sleep(5) yield scrapy.Request( self.articleUrl.format(currentPage), method='GET', callback=self.parseArticlePageJson, #headers=self.headers, cookies=cookie, meta={ 'cookie': cookie, 'currentPage': currentPage, 'totalPage': totalPage, 'beginFlag': beginFlag, 'account': account })
def parseVideoAuthor(self, response): if response.status != 200: print('get url error: ' + response.url) return targetId = response.meta['targetId'] rltJson = json.loads(response.text) if rltJson['errno'] != 0: return fansCnt = rltJson['data']['response']['cnt']['fansCnt'] accountItem = AccountItem() curTime = dateUtil.getCurDate() accountItem['channel_id'] = self.channel_id accountItem['record_class'] = 'channel_info' accountItem['crawl_time'] = curTime accountItem['total_subscribe_count'] = fansCnt print(accountItem)
def parseFansAnalysisPageJson(self, response): if response.status != 200: print('get url error: ' + response.url) return account = response.meta['account'] rltJson = json.loads(response.text) accountItem = AccountItem() accountItem['channel_id'] = self.channel_id accountItem['account_id'] = account accountItem['record_class'] = "channel_info" accountItem['crawl_time'] = dateUtil.getCurDate() accountItem['new_visit_count'] = rltJson['access'] accountItem['total_visit_count'] = rltJson['total_access'] accountItem['new_subscribe_count'] = rltJson['subscribe'] accountItem['total_subscribe_count'] = rltJson['total_subscribe'] yield accountItem
def parseArticlePage(self, response): if response.status != 200: print('get url error: ' + response.url) return rltJson = json.loads(response.text) if self.beginFlag: self.totalPage = rltJson['data']['totalPages'] self.beginFlag = False contentList = rltJson['data']['ldata'] curTime = dateUtil.getCurDate() for contentInfo in contentList: contentItem = ContentItem() contentItem['channel_id'] = self.channel_id contentItem['record_class'] = "content_info" contentItem['crawl_time'] = curTime contentItem['id'] = contentInfo['id'] title = contentInfo['articleTitle'] contentItem['title'] = title contentItem['content_link'] = self.viewArticleUrl.format( contentInfo['id']) if title in self.articleAnalysisDict: contentItem['publish_time'] = self.articleAnalysisDict[title][ 'publish_time'] contentItem['read_count'] = self.articleAnalysisDict[title][ 'read_count'] contentItem['share_count'] = self.articleAnalysisDict[title][ 'share_count'] contentItem['collect_count'] = self.articleAnalysisDict[title][ 'collect_count'] else: contentItem['publish_time'] = time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime((int(contentInfo['operTime'])) / 1000)) contentItem['read_count'] = 0 contentItem['share_count'] = 0 contentItem['collect_count'] = 0 yield contentItem self.curPage += 1 if self.curPage <= self.totalPage: yield FormRequest(self.articleUrl, method='POST', formdata={'currPage': str(self.curPage)}, callback=self.parseArticlePage, headers=self.headers, cookies=self.cookies)
def parseListPageJson(self, response): if response.status != 200: print('get url error: ' + response.url) return rltJson = json.loads(response.text) if self.beginFlag: self.totalNumber = rltJson['data']['totalNumber'] self.maxIndexId = math.ceil(self.totalNumber / 10) self.beginFlag = False contentList = rltJson['data']['articles'] curTime = dateUtil.getCurDate() for contentInfo in contentList: contentItem = ContentItem() contentItem['channel_id'] = self.channel_id #contentItem['account_id'] = "2991941540" #######test contentItem['record_class'] = "content_info" contentItem['crawl_time'] = curTime contentItem['id'] = contentInfo['article_id'] contentItem['title'] = contentInfo['title'] contentItem['content_link'] = contentInfo['url'] contentItem['publish_time'] = contentInfo['pub_time'] contentItem['comment_count'] = contentInfo['commentnum'] if 'vid' in contentInfo: vid = contentInfo['vid'] yield scrapy.Request(self.videoDetailUrl.format(vid), callback=self.parseVideoDetailPageJson, method='GET', headers=self.headers, cookies=self.cookies, meta={'item': contentItem}) else: yield scrapy.Request( self.articleDetailUrl.format(contentItem['id'] + '00'), callback=self.parseArticleDetailPageJson, method='GET', headers=self.headers, cookies=self.cookies, meta={'item': contentItem}) self.indexId += 1 if self.indexId <= self.maxIndexId: yield scrapy.Request(self.contentListStartUrl.format(self.indexId), callback=self.parseListPageJson, method='GET', headers=self.headers, cookies=self.cookies)
def parseChannelPage(self, response): if response.status != 200: print('get url error: ' + response.url) return rltJson = json.loads(response.text) accountItem = AccountItem() accountItem['channel_id'] = self.channel_id accountItem['record_class'] = "channel_info" accountItem['crawl_time'] = dateUtil.getCurDate() accountItem['new_subscribe_count'] = rltJson['data'][0][ 'subscribe_cnt'] accountItem['total_subscribe_count'] = rltJson['data'][0][ 'his_subscribe_cnt'] accountItem['cancel_fans_count'] = rltJson['data'][0][ 'cancel_subscribe_cnt'] yield accountItem
def parseArticlePageJson(self, response): if response.status != 200: print('get url error: ' + response.url) return account = response.meta['account'] token = response.meta['token'] currentPage = response.meta['currentPage'] totalPage = response.meta['totalPage'] beginFlag = response.meta['beginFlag'] rltJson = json.loads(response.text) if beginFlag: totalPage = rltJson['data']['total_page'] beginFlag = False contentList = rltJson['data']['data'] curTime = dateUtil.getCurDate() for contentInfo in contentList: contentItem = ContentItem() contentItem['channel_id'] = self.channel_id contentItem['account_id'] = account contentItem['record_class'] = "content_info" contentItem['crawl_time'] = curTime contentItem['id'] = contentInfo['id'] contentItem['title'] = contentInfo['title'] contentItem['content_link'] = contentInfo['url'] contentItem['publish_time'] = contentInfo['publish_time'] contentItem['read_count'] = contentInfo['pv'] contentItem['comment_count'] = contentInfo['comment_num'] contentItem['share_count'] = contentInfo['share_num'] contentItem['collect_count'] = contentInfo['fav_num'] contentItem['recommend_count'] = contentInfo['rec_show_pv'] status = int(contentInfo['status']) #趣头条:1-草稿;5-待审核;2-已发布;3-审核失败;4-回收站 contentItem['publish_status'] = publicContentStatus[channelContentStatus[self.channel_id]['article'][status]] yield contentItem currentPage += 1 if currentPage <= totalPage: time.sleep(5) yield scrapy.Request( self.articleUrl.format(currentPage, token, self.dtu), method='GET', callback=self.parseArticlePageJson, meta={'token': token, 'currentPage': currentPage, 'totalPage': totalPage, 'beginFlag': beginFlag, 'account': account})
def parseContentPageJson(self, response): if response.status != 200: print('get url error: ' + response.url) return rltJson = json.loads(response.text) if self.beginFlag: self.totalPage = rltJson['data']['page']['totalPage'] self.beginFlag = False contentList = rltJson['data']['list'] curTime = dateUtil.getCurDate() for contentInfo in contentList: contentItem = ContentItem() contentItem['channel_id'] = self.channel_id #contentItem['account_id'] = "13656689260" #######test contentItem['record_class'] = "content_info" contentItem['crawl_time'] = curTime contentItem['id'] = contentInfo['id'] contentItem['title'] = contentInfo['title'] contentItem['content_link'] = contentInfo['url'] contentItem['publish_time'] = contentInfo['publish_time'] contentItem['audit_result'] = contentInfo['audit_msg'] contentItem['read_count'] = contentInfo['read_amount'] contentItem['comment_count'] = contentInfo['comment_amount'] contentItem['share_count'] = contentInfo['share_amount'] contentItem['collect_count'] = contentInfo['collection_amount'] contentItem['recommend_count'] = contentInfo['rec_amount'] contentItem['like_count'] = contentInfo['like_amount'] status = contentInfo['status'] contentItem['publish_status'] = publicContentStatus[channelContentStatus[self.channel_id][status]] '''if status == 'publish': contentItem['publish_status'] = 3 if status == 'rejected': contentItem['publish_status'] = 2''' yield contentItem self.currentPage += 1 if self.currentPage <= self.totalPage: yield scrapy.Request(self.contentStartUrl.format(self.currentPage), callback=self.parseContentPageJson, method='GET', headers=self.headers, cookies=self.cookies)
def parseVideoInfo(self, response): if response.status != 200: print('get url error: ' + response.url) return account = response.meta['account'] url = response.url id = url.split("/")[-1] uploadDate = response.meta['uploadDate'] titleList = response.xpath('//h1[starts-with(@class,"detail-cover-title")]/text()').extract() title = "" if len(titleList) == 1: title = titleList[0].strip() publishTimeList = response.xpath('//div[@itemprop="datePublished"]/strong/text()').extract() temp_publish_time = "" if len(publishTimeList) == 1: temp_publish_time = publishTimeList[0].strip() if temp_publish_time.find("分钟前") >= 0: try: minute_count = int(temp_publish_time.replace("分钟前", "")) except: minute_count = 0 publish_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()-minute_count*60)) else: temp_publish_time += ":00" temp_publish_time = temp_publish_time.replace("今天", "") temp_time = temp_publish_time.split(" ")[-1] publish_time = uploadDate + " " + temp_time playCountList = response.xpath('//div[@class="detail-location"]/text()').extract() play_count = 0 if len(playCountList) == 2: try: play_count = int(playCountList[-1].strip().replace("播放", "")) except: play_count = 0 likeCountList = response.xpath('//span[@itemprop="ratingCount"]/text()').extract() like_count = 0 if len(likeCountList) == 1: try: like_count = int(likeCountList[0].strip()) except: like_count = 0 commentCountList = response.xpath('//span[@itemprop="reviewCount"]/text()').extract() comment_count = 0 if len(commentCountList) == 1: try: comment_count = int(commentCountList[0].strip()) except: comment_count = 0 shareCountList = response.xpath('//span[@class="pr top-3"]/text()').extract() share_count = 0 if len(shareCountList) == 2: try: share_count = int(shareCountList[1].strip().replace("分享", "")) except: share_count = 0 curTime = dateUtil.getCurDate() contentItem = ContentItem() contentItem['channel_id'] = self.channel_id contentItem['account_id'] = account contentItem['record_class'] = "content_info" contentItem['crawl_time'] = curTime contentItem['id'] = id contentItem['title'] = title contentItem['content_link'] = url contentItem['publish_time'] = publish_time contentItem['read_count'] = play_count contentItem['comment_count'] = comment_count contentItem['share_count'] = share_count contentItem['like_count'] = like_count yield contentItem