def category_parse(self, response):
items = []
try:
page = response.request.meta['page'] if 'page' in response.request.meta else 1
page = int(page)
if page > int(self.max_search_page):
return items
kw_id = response.request.meta['kw_id'] if 'kw_id' in response.request.meta else None
pg_id = response.request.meta['pg_id'] if 'pg_id' in response.request.meta else None
cat_id = response.request.meta['cat_id'] if 'cat_id' in response.request.meta else None
subject_id = response.request.meta['subject_id'] if 'subject_id' in response.request.meta else None
url = response.request.url
body = response.body
#子分类
results = iqiyi_url_extract.category_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.category_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#剧集
results = iqiyi_url_extract.video_set_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.video_set_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#标签
results = iqiyi_url_extract.tags_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.tags_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#用户
results = iqiyi_url_extract.user_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.user_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#播放
results = iqiyi_url_extract.video_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.video_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#下一页
next_pages = response.xpath('//div[@class="mod-page"]//a[@data-key="down"]/@href').extract()
if next_pages:
for href in next_pages:
if href.startswith('/'):
#list.iqiyi.com
if url.startswith(self.list_url_prefix):
href = self.list_url_prefix + href
#www.iqiyi.com/lib/
elif url.startswith(self.iqiyi_url_prefix):
href = self.iqiyi_url_prefix + href
items.append(Request(url=href, callback=self.category_parse, meta={'page': page+1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
except Exception, e:
log.msg(traceback.format_exc(), level=log.ERROR)
def channel_parse(self, response):
items = []
try:
page = response.request.meta['page'] if 'page' in response.request.meta else 1
page = int(page)
if page > int(self.max_search_page):
return items
kw_id = response.request.meta['kw_id'] if 'kw_id' in response.request.meta else None
pg_id = response.request.meta['pg_id'] if 'pg_id' in response.request.meta else None
cat_id = response.request.meta['cat_id'] if 'cat_id' in response.request.meta else None
subject_id = response.request.meta['subject_id'] if 'subject_id' in response.request.meta else None
url = response.request.url
body = response.body
#频道
results = iqiyi_url_extract.channel_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.channel_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#分类
results = iqiyi_url_extract.category_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.category_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#剧集
results = iqiyi_url_extract.video_set_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.video_set_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#标签
results = iqiyi_url_extract.tags_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.tags_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#用户
results = iqiyi_url_extract.user_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.user_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#播放
results = iqiyi_url_extract.video_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.video_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
except Exception, e:
log.msg(traceback.format_exc(), level=log.ERROR)
def tags_parse(self, response):
items = []
try:
page = response.request.meta['page'] if 'page' in response.request.meta else 1
page = int(page)
if page > int(self.max_search_page):
return items
kw_id = response.request.meta['kw_id'] if 'kw_id' in response.request.meta else None
pg_id = response.request.meta['pg_id'] if 'pg_id' in response.request.meta else None
cat_id = response.request.meta['cat_id'] if 'cat_id' in response.request.meta else None
subject_id = response.request.meta['subject_id'] if 'subject_id' in response.request.meta else None
url = response.request.url
body = response.body
#用户
results = iqiyi_url_extract.user_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.user_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#播放
results = iqiyi_url_extract.video_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.video_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#下一页
#获取页数
total_nums = 0
perpage_nums = 0
total_nums_str = response.xpath('//div[@class="mod-page"]/@data-huati-total').extract()
if total_nums_str:
total_nums = float(total_nums_str[0])
perpage_nums_str = response.xpath('//div[@class="mod-page"]/@data-huati-perpage').extract()
if perpage_nums_str:
perpage_nums = float(perpage_nums_str[0])
if perpage_nums != 0:
page_nums = int(math.ceil(total_nums/perpage_nums))
if page_nums >= 2:
i = 2
while i <= page_nums:
url = url.repalce('.html', '/%s.html'%i)
items.append(Request(url=url, callback=self.tags_parse, meta={'page': i, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
i = i+1
except Exception, e:
log.msg(traceback.format_exc(), level=log.ERROR)
def page_parse(self, response):
items = []
try:
page = response.request.meta['page'] if 'page' in response.request.meta else 1
page = int(page)
if page > int(self.max_search_page):
return items
kw_id = response.request.meta['kw_id'] if 'kw_id' in response.request.meta else None
pg_id = response.request.meta['pg_id'] if 'pg_id' in response.request.meta else None
cat_id = response.request.meta['cat_id'] if 'cat_id' in response.request.meta else None
subject_id = response.request.meta['subject_id'] if 'subject_id' in response.request.meta else None
url = response.request.url
protocol, rest = urllib2.splittype(url)
if not protocol:
protocol = 'http'
rest = '//' + rest
if rest:
domain, rest = urllib2.splithost(rest)
if domain:
url_prefix = protocol + '://' + domain
else:
url_prefix = self.iqiyi_url_prefix
body = response.body
#用户
results = iqiyi_url_extract.user_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.user_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#播放
results = iqiyi_url_extract.video_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.video_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#下一页
next_pages = response.xpath('//div[@class="mod-page"]//a[@data-key="down"]/@href').extract()
if next_pages:
for href in next_pages:
if href.startswith('/'):
href = url_prefix + href
items.append(Request(url=href, callback=self.page_parse, meta={'page': page+1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
except Exception, e:
log.msg(traceback.format_exc(), level=log.ERROR)
def video_set_parse(self, response):
items = []
try:
kw_id = response.request.meta['kw_id'] if 'kw_id' in response.request.meta else None
pg_id = response.request.meta['pg_id'] if 'pg_id' in response.request.meta else None
cat_id = response.request.meta['cat_id'] if 'cat_id' in response.request.meta else None
subject_id = response.request.meta['subject_id'] if 'subject_id' in response.request.meta else None
url = response.request.url
body = response.body
#模拟ajax获取更多数据
#http://www.iqiyi.com/a_19rrguehx9.html#vfrm=2-3-0-1
regex_express = 'http://www\.iqiyi\.com/a_[\w]+\.html.*'
regex_pattern = re.compile(regex_express)
match = regex_pattern.search(url)
if match:
cache_url_prefix = 'http://cache.video.qiyi.com/jp/sdvlst'
scripts = response.xpath('//script[@type="text/javascript"]')
categoryId = scripts.re('cid:[ ]*(\d+)')
sourceId = scripts.re('sourceId:[ ]*(\d+)')
years = response.xpath('//div[@id="block-J"]//div[@class="choose-y-bd"]//a/@data-year').extract()
months = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
for year in years:
log('year:%s'%year)
for month in months:
log('month:%s'%month)
tvYear = '%s%s'%(year, month)
url = cache_url_prefix + '/%s/%s/%s?categoryId=%s&sourceId=%s&tvYear=%s'%(categoryId, sourceId, tvYear, categoryId, sourceId, tvYear)
items.append(Request(url=url, callback=self.video_set_parse_halfjson, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#http://www.iqiyi.com/lib/s_214734305.html#vfrm=2-3-0-1
regex_express = 'http://www\.iqiyi\.com/lib/s_([\w]+)\.html.*'
regex_pattern = re.compile(regex_express)
match = regex_pattern.search(url)
if match:
video_url_prefix = 'http://rq.video.iqiyi.com/star/s/w.jsonp'
results = match.groups()
id = results[0]
page = 1
cids = response.xpath('//a[@class="more"]/@data-cid').extract()
if cids:
cids = set(cids)
for cid in cids:
url = video_url_prefix + '?id=%s&t=%s&page=%s'%(id, cid, page)
items.append(Request(url=url, callback=self.video_set_parse_json, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#http://www.iqiyi.com/lib/m_200047914.html
regex_express = 'http://www\.iqiyi\.com/lib/m_[\w]+\.html.*'
regex_pattern = re.compile(regex_express)
match = regex_pattern.search(url)
if match:
title = response.xpath('//div[@class="result_pic"]//a/@title').extract()
video_url_prefix = 'http://rq.video.iqiyi.com/aries/t/l.fjsonp'
url = video_url_prefix + '?page=%s&title=%s'%(page, title)
items.append(Request(url=url, callback=self.video_set_parse_nojson, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#http://www.iqiyi.com/dianshiju/playlist295748402.html#vfrm=2-3-0-1
#该类型的视频集获取用户与播放即可
#用户
results = iqiyi_url_extract.user_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.user_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
#播放
results = iqiyi_url_extract.video_url_extract(url, body)
if results:
for result in results:
items.append(Request(url=result, callback=self.video_parse, meta={'page':1, 'kw_id':kw_id, 'pg_id':pg_id, 'cat_id':cat_id, 'subject_id':subject_id}))
except Exception, e:
log.msg(traceback.format_exc(), level=log.ERROR)