def parse_page(self, response):
try:
logging.log(logging.INFO, 'page:%s' % response.request.url)
cat = response.request.meta['cat']
items = []
qq_v = response.xpath('//div[@class="mod_cont"]/ul/li')
for v in qq_v:
urls = v.xpath('./h6/a/@href').extract()
titles = v.xpath('./h6/a/@text').extract()
thumb_urls = v.xpath('./a/img/@src').extract()
durations = v.xpath(
'./a/div/span[@class="mod_version"]/text()').extract()
playeds = v.xpath('./p/span/text()').extract()
title = titles[0] if titles else None
thumb_url = thumb_urls[0] if thumb_urls else None
duration = Util.get_qq_duration(
durations[0]) if durations else None
played = Util.normalize_played(Util.normalize_vp(
playeds[0])) if playeds else None
if urls:
r = Request(url=urls[0], callback=self.parse_episode)
d = {
'title': title,
'thumb_url': thumb_url,
'duration': duration,
'played': played
}
d.update(order)
r.meta.update({'order': d})
items.append(r)
return items
except Exception as e:
logging.log(logging.ERROR, traceback.format_exc())
def parse_page(self, response):
try:
logging.log(logging.INFO, 'page:%s' % response.request.url)
order = response.request.meta['order']
items = []
qq_v = response.xpath(
'//ul[@id="videolst_cont"]/li[@class="list_item"]')
for v in qq_v:
urls = v.xpath('./strong/a/@href').extract()
titles = v.xpath('./strong/a/text()').extract()
thumb_urls = v.xpath('./a/img/@src').extract()
durations = v.xpath('./a/span/em/text()').extract()
playeds = v.xpath(
'./div/span[@class="figure_info_play"]/span/text()'
).extract()
upload_times = v.xpath(
'./div/span[@class="figure_info_time"]/text()').extract()
title = titles[0] if titles else None
thumb_url = thumb_urls[0] if thumb_urls else None
duration = Util.get_qq_duration(
durations[0]) if durations else None
played = Util.normalize_played(Util.normalize_vp(
playeds[0])) if playeds else None
upload_time = Util.get_qq_upload_time(
upload_times[0]) if upload_times else None
if urls:
r = Request(url=urls[0], callback=self.parse_episode)
d = {
'title': title,
'thumb_url': thumb_url,
'duration': duration,
'played': played,
'upload_time': upload_time
}
d.update(order)
r.meta.update({'order': d})
items.append(r)
return items
except Exception as e:
logging.log(logging.ERROR, traceback.format_exc())