def parse_list(self,response):
content = json.loads(response.body)
datas = content.get('datas')
if datas:
echarts = datas.get('echarts')
if echarts:
echarts = echarts[0]
meta = response.meta
channelList = echarts.get('channelList')
if channelList:
item = QianfanDetailItem()
item['content'] = content
item['meta'] = meta
yield item
date_ = meta['date_']
if date_==start_date:
timeRange = datas.get('timeRange')
for tr in timeRange:
date_1 = tr.get('key')
url = response.url
url = url.replace(date_,date_1)
meta['date_'] = date_1
meta['retry_times'] = 0
meta['url'] = url
yield Request(url, callback=self.parse_list, meta=meta)
def parse(self, response):
item = QianfanDetailItem()
content = response.body
item['content'] = content
item['meta'] = response.meta
yield item
def parse(self, response):
content = response.body
content_json = json.loads(content)
datas = content_json.get('datas')
if datas:
echarts = datas.get('echarts')
if echarts:
next_page_sign = 0
echarts = echarts[0]
meta = response.meta
for echart_key, echart_val in echarts.items():
if 'app' not in echart_key:
if echart_val:
next_page_sign = 1
item = QianfanDetailItem()
item['content'] = content
item['meta'] = meta
yield item
break
if next_page_sign:
date_ = meta['date_']
now = datetime.datetime.strptime(date_, '%Y/%m/%d')
date_1 = (now.replace(day=1) -
datetime.timedelta(1)).replace(day=1)
date_1 = date_1.strftime('%Y/%m/%d')
url = response.url.replace(date_, date_1)
print url
meta['date_'] = date_1
meta['retry_times'] = 0
meta['url'] = url
yield Request(url, callback=self.parse, meta=meta)
def parse(self, response):
item = QianfanDetailItem()
content = response.body
content_json = json.loads(content)
datas = content_json.get('datas')
if datas:
echarts = datas.get('echarts')
if echarts:
echarts = echarts[0]
indexList = echarts.get('indexList')
if indexList:
item['content'] = content
item['meta'] = response.meta
yield item
def parse(self, response):
item = QianfanDetailItem()
content = response.body
content_json = json.loads(content)
datas = content_json.get('datas')
if datas:
appRankList = datas.get('appRankList')
if appRankList:
appRankList = appRankList[0]
cateRanks = appRankList.get('cateRanks')
if cateRanks:
item['content'] = content
item['meta'] = response.meta
yield item