def firstSpider(self, seed):
name = ""
poster = ""
star = ""
director = ""
ctype = ""
programLanguage = ""
intro = ""
mainId = ""
area = ""
shootYear = ""
doc = spiderTool.getHtmlBody(seed)
tvId = re.search(r'data-player-tvid="([^"]+?)"', doc)
videoId = re.search(r'data-player-videoid="([^"]+?)"', doc)
if tvId and videoId:
newUrl = 'http://cache.video.qiyi.com/vi/%s/%s/' % (tvId.group(1), videoId.group(1))
doc = spiderTool.getHtmlBody(newUrl)
else:
return
try:
json_data = json.loads(doc)
name = json_data["shortTitle"]
poster = json_data["apic"]
star = json_data["ma"].replace("|", ",")
director = json_data["d"].replace("|", ",")
ctype = json_data["tg"].replace(" ", ",")
programLanguage = json_data["ar"]
intro = json_data["info"]
except:
return
self.program["name"] = spiderTool.changeName(name)
self.program['poster'] = spiderTool.listStringToJson('url',poster)
self.program['star'] = spiderTool.listStringToJson('name',star)
self.program['director'] = spiderTool.listStringToJson('name',director)
self.program['ctype'] = spiderTool.listStringToJson('name',ctype)
self.program['programLanguage'] = programLanguage
self.program['intro'] = intro
if re.match(r'http://www\.iqiyi\.com/v_(\w+)\.html', seed):
mainId = re.match(r'http://www\.iqiyi\.com/v_(\w+)\.html', seed).group(1)
elif re.match(r'http://www\.iqiyi\.com/dianying/(\d{8})/', seed):
mainId = re.match(r'http://www\.iqiyi\.com/dianying/(\d{8})/', seed).group(1)
self.program["mainId"] = mainId
self.program_sub = copy.deepcopy(PROGRAM_SUB)
self.program_sub['setNumber'] = 1
self.program_sub['setName'] = name
self.program_sub['webUrl'] = seed
self.program['programSub'].append(self.program_sub)
def run(self):
getUrl = "http://localhost:9527/spider/cronData?runTime=" + self.runTime
data = spiderTool.getHtmlBody(getUrl)
if data == "":
return ""
dataList = json.loads(data)
modelList = os.listdir(spiderModelPath)
for each in dataList:
spider = each['spider']
url = each['url']
if spider == "" or url == "":
continue
if spider + ".py" not in modelList:
continue
else:
try:
spider_module = __import__(spider)
spider_class = eval("spider_module." + spider + "()")
except: # IOError, e:
spider_class = ""
if spider_class != "":
try:
spider_class.seedList = [url]
spider_class.seedSpider()
jsonData = spider_class.jsonData
except: # IOError, e:
jsonData = ""
else:
jsonData = ""
if jsonData != "":
self.openFile.write(str(jsonData))
def secondSpider(self, seed):
data = ""
doc = spiderTool.getHtmlBody(seed)
json_data = json.loads(doc.split('videoList(')[1][:-2])
if type(json_data) is types.DictionaryType and json_data.has_key("data"):
data_P = json_data["data"]
if type(data_P) is types.DictionaryType and data_P.has_key("list"):
data = data_P["list"]
if type(data) is not types.ListType or data == "":
return
for each in data:
self.program_sub = copy.deepcopy(PROGRAM_SUB)
setNumber = ""
setName = ""
webUrl = ""
poster = ""
if each.has_key("title"):
setNumber = each['title']
if each.has_key("title"):
setName = self.program['name']
if each.has_key("url"):
webUrl = each['url']
if each.has_key("capture"):
poster = each['capture']
if setNumber == "" and re.search(r'^\d{6}'.decode('utf8'), setName):
setNumber = re.search(r'^\d{6}', setName).group()
if re.search(r'预告'.decode('utf8'), setName) or setNumber == "" or setName == "" or webUrl == ""\
or setNumber is None or setName is None or webUrl is None:
continue
self.program_sub['setNumber'] = setNumber.replace('-', '')
self.program_sub['setName'] = setName
self.program_sub['webUrl'] = webUrl
self.program_sub['poster'] = poster
self.program['programSub'].append(self.program_sub)
def secondSpider(self, seed):
doc = spiderTool.getHtmlBody(seed)
try:
json_data = json.loads(doc.split('=')[1])
data = json_data["data"]
except:
data = []
for each in data:
self.program_sub = copy.deepcopy(PROGRAM_SUB)
setNumber = each['tvYear']
setName = each['videoName']
webUrl = each['vUrl']
poster = each['aPicUrl']
playLength = each['timeLength']
setIntro = each['desc']
if re.search(r'预告'.decode('utf8'), setName) or setNumber == "" or setName == "" or webUrl == ""\
or setNumber is None or setName is None or webUrl is None:
continue
self.program_sub['setNumber'] = setNumber.replace('-', '')
self.program_sub['setName'] = setName
self.program_sub['webUrl'] = webUrl
self.program_sub['poster'] = poster
self.program_sub['playLength'] = str(
int(playLength) / 60) + ':' + str(int(playLength) % 60)
self.program_sub['setIntro'] = setIntro
self.program['programSub'].append(self.program_sub)
def secondSpider(self, seed):
doc = spiderTool.getHtmlBody(seed)
sub_soup = BeautifulSoup(doc, from_encoding="utf8")
sub_num = sub_soup.find_all("episode_number")
for each in sub_num:
self.program_sub = copy.deepcopy(PROGRAM_SUB)
setNumber = ""
setName = ""
webUrl = ""
poster = ""
setNumber = each.get_text()
setName_p = each.find_next_sibling("title")
if setName_p is not None:
setName = setName_p.get_text()
setUrl_p = each.find_next_sibling("url")
if setUrl_p is not None:
webUrl = setUrl_p.get_text()
poster_p = each.find_next_sibling("pic")
if poster_p is not None:
poster = poster_p.get_text()
if re.search(r'预告'.decode('utf8'), setName) or setNumber == "" or setName == "" or webUrl == ""\
or setNumber is None or setName is None or webUrl is None:
continue
self.program_sub['setNumber'] = setNumber.replace(
"-".decode('utf8'), "")
self.program_sub['setName'] = setName
self.program_sub['webUrl'] = webUrl
self.program_sub['poster'] = poster
self.program['programSub'].append(self.program_sub)
def secondSpider(self, seed):
doc = spiderTool.getHtmlBody(seed)
sub_soup = BeautifulSoup(doc, from_encoding="utf8")
sub_num = sub_soup.find_all("ul", attrs={'class': 'item'})
for each in sub_num:
self.program_sub = copy.deepcopy(PROGRAM_SUB)
setNumber = ""
setName = ""
webUrl = ""
stars = ""
setName_P = each.find('li')
if setName_P is not None:
a_tag = setName_P.find("a")
if a_tag is not None:
setName = a_tag.get('title')
webUrl = a_tag.get('href')
label_tag = setName_P.find("label")
if label_tag is not None:
num_tag = label_tag.get_text().strip()
if self.program['shootYear'] != "" and re.search(
r'^\d{2}-\d{2}期$'.decode('utf8'), num_tag):
setNumber = self.program['shootYear'] + num_tag.replace(
"-", "").replace("期".decode('utf8'), "")
elif re.search(r'^\d{2}-\d{2}期$'.decode('utf8'), num_tag):
year = time.strftime('%y', time.localtime())
if re.search(r'reload_(20\d{2})', seed):
setNumber = re.search(
r'reload_(20\d{2})',
seed).group(1) + num_tag.replace(
"-", "").replace("期".decode('utf8'), "")
elif re.search(year, setName):
setNumber = time.strftime(
'%Y', time.localtime()) + num_tag.replace(
"-", "").replace("期".decode('utf8'), "")
elif re.search(r'\d+'.decode('utf8'), num_tag):
setNumber = re.search(r'\d+'.decode('utf8'),
num_tag).group()
else:
setNumber = ""
stars_P = each.find('li', attrs={'class': 'iguests'})
if stars_P is not None:
starsList = []
a_tag = stars_P.find_all("a")
for a in a_tag:
if a.get_text() is not None:
starsList.append(a.get_text().strip())
stars = ",".join(starsList)
if re.search(r'预告'.decode('utf8'), setName) or setNumber == "" or setName == "" or webUrl == ""\
or setNumber is None or setName is None or webUrl is None:
continue
self.program_sub['setNumber'] = setNumber
self.program_sub['setName'] = setName
self.program_sub['webUrl'] = webUrl
self.program_sub['stars'] = stars
self.program['programSub'].append(self.program_sub)
def doProcess(self):
for seed in self.seedBase:
doc = spiderTool.getHtmlBody(seed)
soup = BeautifulSoup(doc, from_encoding="utf8")
seedSoup = soup.find_all('a', attrs={'class': 'ui-list-ct'})
for each in seedSoup:
subSeed = each.get("href")
if subSeed is not None:
self.seedList.append(subSeed)
self.seedSpider()
def secondSpider(self, seed, seed_P, pid):
data = []
doc = spiderTool.getHtmlBody(seed)
json_data = json.loads(doc)
if json_data is not None:
if json_data.has_key('data'):
data_p = json_data['data']
if type(data_p) is types.DictionaryType:
if data_p.has_key('list'):
data = data_p['list']
if type(data) is not types.ListType:
return
for each in data:
self.program_sub = copy.deepcopy(PROGRAM_SUB)
setNumber = ""
setName = ""
webUrl = ""
poster = ''
playLength = ""
setIntro = ""
if type(each) is not types.DictionaryType:
return
if each.has_key('videoId'):
videoId = str(each['videoId'])
webUrl = seed_P.replace(pid, videoId)
if each.has_key('videoIndex'):
setNumber = str(each['videoIndex'])
if setNumber == "" and len(data) == 1:
setNumber = '1'
if each.has_key('name'):
setName = each['name']
if each.has_key('image'):
poster = each['image']
if each.has_key('desc'):
setIntro = each['desc']
if each.has_key('time'):
timeLength = each['time'].encode('utf8')
playLength = str(int(timeLength) / 60) + ":" + str(
int(timeLength) % 60)
if re.search(r'预告'.decode('utf8'), setName) or setNumber == "" or setName == "" or webUrl == ""\
or setNumber is None or setName is None or webUrl is None:
continue
self.program_sub['setNumber'] = setNumber.replace('-', '')
self.program_sub['setName'] = setName
self.program_sub['webUrl'] = webUrl
self.program_sub['poster'] = poster
self.program_sub['playLength'] = playLength
self.program_sub['setIntro'] = setIntro
self.program['programSub'].append(self.program_sub)
def secondSpider(self, seed):
data = []
doc = spiderTool.getHtmlBody(seed)
try:
json_data = json.loads(doc)
except:
json_data = {}
if json_data is not None:
if json_data.has_key('body'):
data_p = json_data['body']
if type(data_p) is types.DictionaryType:
if data_p.has_key('videoInfo'):
data = data_p['videoInfo']
if type(data) is not types.ListType:
return
for each in data:
self.program_sub = copy.deepcopy(PROGRAM_SUB)
setNumber = ""
setName = ""
webUrl = ""
poster = ''
if type(each) is not types.DictionaryType:
return
if each.has_key('episode'):
setNumber = each['episode']
if setNumber == "" and len(data) == 1:
setNumber = '1'
if each.has_key('subTitle'):
setName = each['subTitle']
if setName == '' and each.has_key('nameCn'):
setName = each['nameCn']
if each.has_key('picAll'):
poster_P = each['picAll']
if type(poster_P) is types.DictionaryType:
for key in poster_P:
if type(poster_P[key]) is types.UnicodeType:
if poster_P[key] != '':
poster = poster_P[key]
break
if each.has_key('id'):
id = each['id']
if type(id) is types.UnicodeType:
if id != '':
webUrl = 'http://www.le.com/ptv/vplay/%s.html' % (id)
if re.search(r'预告'.decode('utf8'), setName) or setNumber == "" or setName == "" or webUrl == ""\
or setNumber is None or setName is None or webUrl is None:
continue
self.program_sub['setNumber'] = setNumber.replace('-', '')
self.program_sub['setName'] = setName
self.program_sub['webUrl'] = webUrl
self.program_sub['poster'] = poster
self.program['programSub'].append(self.program_sub)
def doProcess(self):
for seed in self.seedBase:
self.seedList = []
seedList = []
doc = spiderTool.getHtmlBody(seed)
soup = BeautifulSoup(doc, from_encoding="utf8")
seed_P = soup.find("div", attrs={"class": "wrapper-piclist"})
if seed_P is not None:
seedList = seed_P.find_all("div", attrs={"class": "site-piclist_pic"})
for each in seedList:
a_tag = each.find("a")
if a_tag is not None:
subSeed = a_tag.get("href")
if subSeed is not None:
self.seedList.append(subSeed)
self.seedSpider()
def doProcess(self):
for seed in self.seedBase:
self.seedList = []
seedList = []
doc = spiderTool.getHtmlBody(seed)
soup = BeautifulSoup(doc, from_encoding="utf8")
seed_P = soup.find("div", attrs={"class": "wrapper-piclist"})
if seed_P is not None:
seedList = seed_P.find_all("div", attrs={"class": "site-piclist_pic"})
for each in seedList:
a_tag = each.find("a")
if a_tag is not None:
subSeed = a_tag.get("href")
if subSeed is not None:
if re.match(r'http://www\.iqiyi\.com/v_\w+\.html', subSeed) or re.match(r'http://www\.iqiyi\.com/dianying/\d{8}/', subSeed):
self.seedList.append(subSeed)
self.seedSpider()
def doProcess(self):
for seed in self.seedBase:
self.seedList = []
seedList = []
doc = spiderTool.getHtmlBody(seed)
soup = BeautifulSoup(doc, from_encoding="utf8")
seed_P = soup.find("ul", attrs={"class": "figures_list"})
if seed_P is not None:
seedList = seed_P.find_all("li", attrs={"class": "list_item"})
for each in seedList:
a_tag = each.find("a")
if a_tag is not None:
subSeed = a_tag.get("href")
if subSeed is not None:
subSeed = re.sub(r'/cover/\w/', '/x/cover/', subSeed)
self.seedList.append(subSeed)
self.seedSpider()
def doProcess(self):
for seed in self.seedBase:
seedList = []
doc = spiderTool.getHtmlBody(seed)
soup = BeautifulSoup(doc, from_encoding="utf8")
seed_P = soup.find("ul", attrs={"class": "clearfix ullist-ele"})
if seed_P is not None:
seedList = seed_P.find_all("p", attrs={"class": "img-box"})
for each in seedList:
a_tag = each.find("a")
if a_tag is not None:
subSeed = a_tag.get("href")
if subSeed is not None:
subSeed = subSeed.replace('www.hunantv.com',
'www.mgtv.com')
self.seedList.append(subSeed)
self.seedSpider()
def mfsp():
form = mfspForm()
form.type.choices = [('1', u'电影'), ('2', u'电视剧'), ('5', u'动漫'),
('4', u'综艺'), ('9', u'纪录片')]
data = []
if form.validate_on_submit():
requestForm = request.form
keyList = requestForm.keys()
word = ""
type = ""
if "word" in keyList:
word = requestForm["word"]
if "type" in keyList:
type = requestForm["type"]
if word != "" and type != "":
seed = "http://www.beevideo.tv/api/video2.0/video_search.action?channelId=%s&searchKey=%s" % (
type, word)
doc = spiderTool.getHtmlBody(seed)
soup = BeautifulSoup(doc, from_encoding="utf8")
video_list_p = soup.find('video_list')
if video_list_p is not None:
video_list = video_list_p.find_all('video_item')
for each in video_list:
item = {}
url = ""
name = ""
type = ""
id = each.find('id')
if id is not None:
id_num = id.get_text()
url = 'http://www.beevideo.tv/api/video2.0/video_detail_info.action?videoId=' + id_num
name_tag = each.find('name')
if name_tag:
name = name_tag.get_text()
type_tag = each.find('channel')
if type_tag:
type = type_tag.get_text()
if name != "" and type != "" and url != "":
item = {"name": name, "type": type, "url": url}
data.append(item)
return render_template('spider/mfsp.html', form=form, data=data)
def doProcess(self):
for seed in self.seedBase:
seedList = []
doc = spiderTool.getHtmlBody(seed)
try:
json_data = json.loads(doc)
except:
json_data = {}
if json_data.has_key("video_list"):
if type(json_data["video_list"]) is types.ListType:
seedList = json_data["video_list"]
for each in seedList:
pid = ""
if each.has_key("aid"):
if type(each["aid"]) is types.UnicodeType:
pid = each["aid"]
subSeed = "http://www.le.com/jilu/" + pid + ".html"
self.seedList.append(subSeed)
if pid != "" and each.has_key("images"):
images = each["images"]
if type(images) is types.DictionaryType:
if images.has_key("600*800"):
self.images[pid] = images["600*800"]
self.seedSpider()
def firstSpider(self, seed):
poster = ""
star = ""
director = ""
ctype = ""
shootYear = ""
intro = ""
mainId = ""
area = ""
name = ""
programLanguage = ""
doc = spiderTool.getHtmlBody(seed)
soup = BeautifulSoup(doc, from_encoding="utf8")
seed = ""
seed_p = soup.find('a', attrs={'class': 'album_title'})
if seed_p is not None:
if seed_p.get("href") is not None:
seed = seed_p.get("href")
if seed == "":
return
doc = spiderTool.getHtmlBody(seed)
soup = BeautifulSoup(doc, from_encoding="utf8")
poster_p = soup.find("img", attrs={'itemprop': 'image'})
if poster_p is not None:
if poster_p.get("src") is not None:
poster = poster_p.get("src")
if not re.search(r'http', poster):
poster = "http:%s" % poster
if poster_p.get("alt") is not None:
name = poster_p.get("alt")
video_type = soup.find("div", attrs={"class": "video_type cf"})
if video_type is not None:
divs = video_type.find_all('div', attrs={"class": "type_item"})
for div in divs:
text = div.get_text()
if re.search(r'地 区:'.decode("utf8"), text):
area_p = div.find('span', attrs={"class": "type_txt"})
if area_p is not None:
area = area_p.get_text()
if re.search(r'语 言:'.decode("utf8"), text):
programLanguage_p = div.find('span',
attrs={"class": "type_txt"})
if programLanguage_p is not None:
programLanguage = programLanguage_p.get_text()
if re.search(r'上映时间:'.decode("utf8"), text):
shootYear_p = div.find('span', attrs={"class": "type_txt"})
if shootYear_p is not None:
shootYear = shootYear_p.get_text().split("-")[0]
ctype_P = soup.find("div", attrs={"class": "tag_list"})
if ctype_P is not None:
ctype_list = []
for each in ctype_P.find_all('a'):
ctype_list.append(each.get_text())
ctype = ",".join(ctype_list)
person_p = soup.find("ul", attrs={"class": "actor_list cf"})
if person_p is not None:
lis = person_p.find_all('li')
director_list = []
star_list = []
for li in lis:
person = ""
star_p = li.find("span", attrs={'class': 'name'})
if star_p is not None:
person = star_p.get_text()
if person != "" and li.find("span",
attrs={"class": "director"}):
director_list.append(person)
elif person != "":
star_list.append(person)
star = ",".join(star_list)
director = ",".join(director_list)
content_p = soup.find('span',
attrs={"class": "txt _desc_txt_lineHight"})
if content_p is not None:
intro = content_p.get_text().strip()
if re.match(r'http://film\.qq\.com/(page|cover)/.*/(.*?).html', seed):
mainId = re.match(
r'http://film\.qq\.com/(page|cover)/.*/(.*?).html',
seed).group(2)
if re.match(r'http://v\.qq\.com/detail/[\w\d]/(.*?).html', seed):
mainId = re.match(r'http://v\.qq\.com/detail/[\w\d]/(.*?).html',
seed).group(1)
self.program["name"] = spiderTool.changeName(name)
self.program['poster'] = spiderTool.listStringToJson('url', poster)
self.program['star'] = spiderTool.listStringToJson('name', star)
self.program['director'] = spiderTool.listStringToJson(
'name', director)
self.program['ctype'] = spiderTool.listStringToJson('name', ctype)
self.program['shootYear'] = shootYear
self.program['area'] = spiderTool.listStringToJson('name', area)
self.program['programLanguage'] = programLanguage
self.program['intro'] = intro
self.program['mainId'] = mainId
if mainId != "":
seed_sub = 'http://s.video.qq.com/loadplaylist?vkey=898_qqvideo_clpl_%s_qq&vtype=' % mainId
self.secondSpider(seed_sub)
def firstSpider(self, seed):
name = ""
poster = ""
star = ""
director = ""
ctype = ""
programLanguage = ""
intro = ""
mainId = ""
area = ""
doc = spiderTool.getHtmlBody(seed)
soup = BeautifulSoup(doc, from_encoding="utf8")
poster_p = soup.find("img", attrs={'width': '195'})
poster_p_1 = soup.find(
'div', attrs={'class': 'album-picCon album-picCon-onePic'})
poster_p_2 = soup.find('ul', attrs={'class': 'album-imgs'})
if poster_p is not None:
poster = poster_p.get("src")
name = poster_p.get("alt")
elif poster_p_1 is not None:
img_tag = poster_p_1.find("img")
if img_tag is not None:
poster = img_tag.get("src")
name = img_tag.get("alt")
elif poster_p_2 is not None:
poster = poster_p_2.find('img').get('src')
if name == "":
name_p = soup.find('div',
attrs={'class': 'album-playArea clearfix'})
if name_p is not None:
name = name_p.find('h1').get_text()
detail = soup.find("div", attrs={"class": "result_detail-minH"})
detail_1 = soup.find("div", attrs={"class": "msg-hd-lt fl"})
if detail is not None:
for div_p in detail.find_all(
"div", attrs={"class": "topic_item clearfix"}):
for each in div_p.find_all("div"):
a_list = []
for a_tag in each.find_all('a'):
a_list.append(a_tag.get_text())
a_str = ",".join(a_list)
if re.search("主持人：".decode("utf8"), each.get_text()):
star = a_str
if re.search("类型：".decode("utf8"), each.get_text()):
ctype = a_str
if re.search("语言：".decode("utf8"), each.get_text()):
programLanguage = a_str
if re.search("地区：".decode("utf8"), each.get_text()):
area = a_str
elif detail_1:
for each in detail_1.find_all("p"):
a_list = []
for a_tag in each.find_all('a'):
a_list.append(a_tag.get_text())
a_str = ",".join(a_list)
if re.search("主持人：".decode("utf8"), each.get_text()):
star = a_str
if re.search("类型：".decode("utf8"), each.get_text()):
ctype = a_str
if re.search("语言：".decode("utf8"), each.get_text()):
programLanguage = a_str
if re.search("地区：".decode("utf8"), each.get_text()):
area = a_str
content_p = soup.find('span',
attrs={
"class": "showMoreText",
"data-moreorless": "moreinfo",
"style": "display: none;"
})
content_p_1 = soup.find("span", attrs={"class": "bigPic-b-jtxt"})
if content_p is not None:
if content_p.find("span"):
content_p = content_p.find("span")
if re.search("简介：".decode("utf8"), content_p.get_text()):
intro = content_p.get_text().split(
"简介：".decode("utf8"))[1].strip()
elif content_p_1 is not None:
intro = content_p_1.get_text()
self.program["name"] = spiderTool.changeName(name)
self.program['poster'] = spiderTool.listStringToJson('url', poster)
self.program['star'] = spiderTool.listStringToJson('name', star)
self.program['director'] = spiderTool.listStringToJson(
'name', director)
self.program['ctype'] = spiderTool.listStringToJson('name', ctype)
self.program['programLanguage'] = programLanguage
self.program['area'] = spiderTool.listStringToJson('name', area)
self.program['intro'] = intro
seed_P = re.search(
r'sourceId:\s*(?P<sourceId>\d+),\s+cid:\s*(?P<cid>\d+)', doc)
if seed_P:
sourceId = seed_P.group('sourceId')
cid = seed_P.group('cid')
self.program['mainId'] = sourceId
seed_sub = 'http://cache.video.iqiyi.com/jp/sdvlst/%s/%s/' % (
cid, sourceId)
self.secondSpider(seed_sub)
def firstSpider(self, seed):
name = ""
pcUrl = ""
poster = ""
star = ""
director = ""
ctype = ""
programLanguage = ""
intro = ""
point = ""
mainId = ""
area = ""
if re.search(r'http://v.pptv.com/show/\w+\.html', seed):
seed_post = seed.replace("/show/", "/page/")
else:
seed_post = seed
doc = spiderTool.getHtmlBody(seed_post)
soup = BeautifulSoup(doc, from_encoding="utf8")
poster_P = soup.find("a", attrs={"class": "cover-a"})
poster_P1 = soup.find("div", attrs={"class": "bd cf"})
if poster_P is not None:
img_tag = poster_P.find("img")
if img_tag is not None:
img = img_tag.get('data-src2')
if img is not None:
poster = img
elif poster_P1 is not None:
img_tag = poster_P1.find("img")
if img_tag is not None:
img = img_tag.get('src')
if img is not None:
poster = img
if re.search(r'http://v.pptv.com/page/\w+\.html', seed):
seed = seed.replace("/page/", "/show/")
doc = spiderTool.getHtmlBody(seed)
pid_P = re.search(r'"pid":\s*(?P<pid>\d+),'.decode('utf8'), doc)
if pid_P:
pid = pid_P.group('pid')
seed_sub = 'http://v.pptv.com/show/videoList?&cb=videoList&pid=' + pid
mainId = pid
else:
return
id_P = re.search(r'"id":\s*(?P<id>\d+),'.decode('utf8'), doc)
if id_P:
id = id_P.group('id')
seed = 'http://svcdn.pptv.com/show/v1/meta.json?cid=%s&sid=%s&psize=50' % (
id, pid)
else:
return
doc = spiderTool.getHtmlBody(seed)
json_data = json.loads(doc)
if json_data.has_key("data"):
data = json_data["data"]
else:
return
if data.has_key("set"):
name_P = data["set"]
if type(name_P) is types.ListType:
if len(name_P) == 2:
name = name_P[0]
pcUrl = name_P[1]
def getListName(dict_P):
dict_list = []
if type(dict_P) is types.ListType:
for each in dict_P:
if type(each) is not types.DictionaryType:
continue
name = ""
if each.has_key("name"):
name = each["name"]
elif each.has_key("text"):
name = each["text"]
if re.search(r'未知'.decode('utf8'), name) or name == '':
continue
dict_list.append(name)
return ",".join(dict_list)
if data.has_key("directors"):
star = getListName(data["directors"])
if data.has_key("ct"):
ctype = getListName(data["ct"])
if data.has_key("area"):
area = getListName(data["area"])
if data.has_key("score"):
point = data["score"]
try:
point = float(point)
except:
point = 0.0
if data.has_key("summary"):
intro = data["summary"]
self.program["name"] = spiderTool.changeName(name)
self.program['pcUrl'] = pcUrl
self.program['poster'] = spiderTool.listStringToJson('url', poster)
self.program['star'] = spiderTool.listStringToJson('name', star)
self.program['director'] = spiderTool.listStringToJson(
'name', director)
self.program['ctype'] = spiderTool.listStringToJson('name', ctype)
self.program['programLanguage'] = programLanguage
self.program['area'] = spiderTool.listStringToJson('name', area)
self.program['intro'] = intro
self.program['point'] = point
self.program['mainId'] = mainId
self.secondSpider(seed_sub)
def firstSpider(self, seed):
point = 0.0
poster = ""
name = ""
shootYear = ""
alias = ""
area = ""
star = ""
director = ""
ctype = ""
playTimes = 0
intro = ""
mainId = ""
doc = spiderTool.getHtmlBody(seed)
soup = BeautifulSoup(doc, from_encoding="utf8")
seed_Re = re.search(r'http://www\.youku\.com/show_page/id', seed)
if not seed_Re:
seed_P = soup.find('h1', attrs={'class': 'title'})
if seed_P is not None:
seed_aTag = seed_P.find('a')
if seed_aTag is not None:
seed = seed_aTag.get('href')
doc = spiderTool.getHtmlBody(seed)
soup = BeautifulSoup(doc, from_encoding="utf8")
point_P = soup.find('span', attrs={'class': 'rating'})
if point_P is not None:
if point_P.find('em', attrs={'class': 'num'}) is not None:
point = point_P.find('em', attrs={'class': 'num'}).get_text()
poster_p = soup.find("li", attrs={'class': 'thumb'})
if poster_p is not None:
poster = poster_p.find('img').get("src")
name_P = soup.find('h1', attrs={'class': 'title'})
if name_P is not None:
name_span = name_P.find('span', attrs={'class': 'name'})
if name_span is not None:
name = name_span.get_text()
if re.search(r'\d{4}', name):
shootYear = re.search(r'\d+', name).group()
area_P = soup.find('span', attrs={'class': 'area'})
if area_P is not None:
area_list = []
for each in area_P.find_all("a"):
area_list.append(each.get_text())
area = ",".join(area_list)
ctype_all = soup.find_all("span", attrs={"class": "type"})
for ctype_P in ctype_all:
if re.search(r'类型:'.decode('utf8'), ctype_P.get_text()):
ctype_list = []
for each in ctype_P.find_all('a'):
ctype_list.append(each.get_text())
ctype = ",".join(ctype_list)
star_P = soup.find("span", attrs={"class": "actor"})
if star_P is not None:
star_list = []
for each in star_P.find_all('a'):
star_list.append(each.get_text())
star = ",".join(star_list)
director_P = soup.find("span", attrs={"class": "director"})
if director_P is not None:
director_list = []
for each in director_P.find_all('a'):
director_list.append(each.get_text())
director = ",".join(director_list)
playTimes_P = soup.find('span', attrs={'class': 'play'})
if playTimes_P is not None:
playTimesStr = playTimes_P.get_text()
playTimes_list = re.findall(r'(\d+)', playTimesStr)
playTimes = long(''.join(playTimes_list))
content_p = soup.find('span',
attrs={
"class": "short",
'id': 'show_info_short'
})
if content_p is not None:
intro = content_p.get_text().strip()
if re.match(r'http://www\.youku\.com/show_page/(id_(\w+))\.html',
seed):
mainId = re.match(
r'http://www\.youku\.com/show_page/id_(\w+)\.html',
seed).group(1)
self.program["name"] = spiderTool.changeName(name)
self.program["alias"] = spiderTool.changeName(alias)
self.program["point"] = float(point)
self.program['poster'] = spiderTool.listStringToJson('url', poster)
self.program['star'] = spiderTool.listStringToJson('name', star)
self.program['director'] = spiderTool.listStringToJson(
'name', director)
self.program['ctype'] = spiderTool.listStringToJson('name', ctype)
self.program['shootYear'] = shootYear
self.program['area'] = spiderTool.listStringToJson('name', area)
self.program['playTimes'] = long(playTimes)
self.program['intro'] = intro
self.program['mainId'] = mainId
urlList = re.findall(
r'<li\s+data="(reload_\d+)"\s*(class="current"){0,1}\s*>', doc)
if urlList:
for url in urlList:
seed_sub = 'http://www.youku.com/show_episode/id_%s.html?dt=json&divid=%s' % (
mainId, url[0])
self.secondSpider(seed_sub)
else:
seed_sub = 'http://www.youku.com/show_episode_id_%s.html?dt=json' % mainId
self.secondSpider(seed_sub)
def firstSpider(self, seed):
name = ""
poster = ""
point = 0.0
shootYear = ""
star = ""
director = ""
ctype = ""
programLanguage = ""
intro = ""
mainId = ""
area = ""
pid = ""
pid_P = re.search(
r'http://www.mgtv.com/\w/\d+/(?P<mainId>\d+)/\w/(?P<pid>\d+)\.html',
seed)
if pid_P:
pid = pid_P.group('pid')
mainId = pid_P.group('mainId')
seedJson = ""
if pid != '':
seedJson = 'http://m.api.hunantv.com/video/getbyid?videoId=%s' % (
pid)
else:
return
if mainId == "":
mainId = pid
doc = spiderTool.getHtmlBody(seedJson)
if re.search(r'<html>', doc):
doc = spiderTool.getHtmlBody(seedJson)
json_data = json.loads(doc.strip())
if json_data.has_key("data"):
if type(json_data["data"]) is types.DictionaryType:
json_data = json_data["data"]
if json_data.has_key("detail"):
json_data = json_data["detail"]
else:
return
else:
return
if json_data.has_key("collectionName"):
name = json_data["collectionName"]
if json_data.has_key("image"):
poster = json_data["image"]
if json_data.has_key("year"):
shootYear = json_data["year"]
if json_data.has_key("director"):
director_P = json_data["director"]
if type(director_P) is types.UnicodeType:
director = director_P.strip().replace(" / ", ",")
if json_data.has_key("player"):
star_P = json_data["player"]
if type(star_P) is types.UnicodeType:
star = star_P.strip().replace(" / ", ",")
if json_data.has_key("area"):
area_P = json_data["area"]
if type(area_P) is types.UnicodeType:
area = area_P.strip().replace(" ", ",")
if json_data.has_key('desc'):
intro = json_data['desc']
self.program["name"] = spiderTool.changeName(name)
self.program['poster'] = spiderTool.listStringToJson('url', poster)
self.program['point'] = point
self.program['shootYear'] = shootYear
self.program['star'] = spiderTool.listStringToJson('name', star)
self.program['director'] = spiderTool.listStringToJson(
'name', director)
self.program['ctype'] = spiderTool.listStringToJson('name', ctype)
self.program['programLanguage'] = programLanguage
self.program['area'] = spiderTool.listStringToJson('name', area)
self.program['intro'] = intro
self.program['mainId'] = mainId
if json_data.has_key('typeId'):
typeId = json_data['typeId']
if json_data.has_key('collectionId'):
collectionId = json_data['collectionId']
pageNum = 20
if json_data.has_key("totalvideocount"):
pageNum = json_data["totalvideocount"]
seed_sub = "http://m.api.hunantv.com/video/getListV2?videoId=%s&pageId=0&pageNum=%s" % (
pid, pageNum)
try:
self.secondSpider(seed_sub, seed, pid)
except:
print "spider error: " + seed
def firstSpider(self, seed):
name = ""
poster = ""
star = ""
director = ""
ctype = ""
programLanguage = ""
intro = ""
mainId = ""
area = ""
doc = spiderTool.getHtmlBody(seed)
soup = BeautifulSoup(doc, from_encoding="utf8")
poster_p = soup.find("ul", attrs={'class': 'focus_img_list'})
poster_p_1 = soup.find("div", attrs={"class": "result_pic pr"})
if poster_p is not None:
poster_p_sub = poster_p.find('li').get('style')
if re.search(r'\((.*?)\)'.decode('utf8'), poster_p_sub):
poster = re.search(r'\((.*?)\)'.decode('utf8'), poster_p_sub).group(1)
elif poster_p_1 is not None:
img_tag = poster_p_1.find("img")
if img_tag is not None:
poster = img_tag.get("src")
name = img_tag.get("alt")
name_p = soup.find('a', attrs={'class': 'white'})
if name_p is not None:
name = name_p.get_text()
detail = soup.find("div", attrs={"class": "result_detail-minH"})
detail_1 = soup.find("div", attrs={"class": "msg-hd-lt fl"})
if detail is not None:
for div_p in detail.find_all("div", attrs={"class": "topic_item clearfix"}):
for each in div_p.find_all("div"):
a_list = []
for a_tag in each.find_all('a'):
a_list.append(a_tag.get_text())
a_str = ",".join(a_list)
if re.search("主演：".decode("utf8"),each.get_text()):
star = a_str
if re.search("导演：".decode("utf8"),each.get_text()):
director = a_str
if re.search("类型：".decode("utf8"),each.get_text()):
ctype = a_str
if re.search("语言：".decode("utf8"),each.get_text()):
programLanguage = a_str
if re.search("地区：".decode("utf8"),each.get_text()):
area = a_str
elif detail_1 is not None:
for p_tag in detail_1.find_all("p"):
a_list = []
for a_tag in p_tag.find_all('a'):
a_list.append(a_tag.get_text())
a_str = ','.join(a_list)
if re.search("导演：".decode("utf8"),p_tag.get_text()):
director = a_str
if re.search("类型：".decode("utf8"),p_tag.get_text()):
ctype = a_str
if re.search("语言：".decode("utf8"),p_tag.get_text()):
programLanguage = a_str
if re.search("地区：".decode("utf8"),p_tag.get_text()):
area = a_str
if re.search("主演：".decode("utf8"),p_tag.get_text()):
star = a_str
content_p = soup.find('span', attrs={"class": "showMoreText", "data-moreorless":"moreinfo", "style":"display: none;"})
content_p_1 = soup.find('div', attrs={"data-moreorless":"moreinfo"})
if content_p is not None:
if content_p.find("span"):
content_p = content_p.find("span")
if re.search("简介：".decode("utf8"),content_p.get_text()):
intro = content_p.get_text().split("简介：".decode("utf8"))[1].strip()
elif content_p_1 is not None:
if re.search("简介：".decode("utf8"),content_p_1.get_text()):
intro = content_p_1.get_text().split("简介：".decode("utf8"))[1].strip()
self.program["name"] = spiderTool.changeName(name)
self.program['poster'] = spiderTool.listStringToJson('url',poster)
self.program['star'] = spiderTool.listStringToJson('name',star)
self.program['director'] = spiderTool.listStringToJson('name',director)
self.program['ctype'] = spiderTool.listStringToJson('name',ctype)
self.program['programLanguage'] = programLanguage
self.program['area'] = spiderTool.listStringToJson('name', area)
self.program['intro'] = intro
seed_P = re.search(r'albumId:\s*(?P<albumId>\d+)[^\\]*?cid:\s*(?P<cid>\d+)'.decode('utf8'), doc)
if seed_P:
albumId = seed_P.group('albumId')
cid = seed_P.group('cid')
self.program['mainId'] = cid + "_" +albumId
seed_sub = 'http://cache.video.iqiyi.com/jp/avlist/%s/' %(albumId)
allNum = 0
doc = spiderTool.getHtmlBody(seed_sub)
try:
json_data = json.loads(doc.split('=')[1])
data = json_data["data"]["vlist"]
allNum = json_data["data"]["allNum"]
except:
data = []
for i in range(1,(int(allNum)/50 + 2)):
seed_sub = 'http://cache.video.iqiyi.com/jp/avlist/%s/%s/50/' %(albumId, str(i))
self.secondSpider(seed_sub)
def firstSpider(self, seed):
name = ""
poster = ""
point = 0.0
shootYear = ""
star = ""
director = ""
ctype = ""
programLanguage = ""
intro = ""
mainId = ""
area = ""
pid = ""
if re.search(r'http://www\.le\.com/jilu/(?P<pid>\d+)\.html', seed):
pid = re.search(r'http://www\.le\.com/jilu/(?P<pid>\d+)\.html',
seed).group('pid')
elif re.search(r'http://www\.letv\.com/jilu/(?P<pid>\d+)\.html', seed):
pid = re.search(r'http://www\.letv\.com/jilu/(?P<pid>\d+)\.html',
seed).group('pid')
if pid != '' or pid != '0':
seed = 'http://static.app.m.letv.com/android/mod/mob/ctl/album/act/detail/id/%s/pcode/010110014/version/5.2.3.mindex.html' % (
pid)
else:
return
mainId = pid
doc = spiderTool.getHtmlBody(seed)
try:
json_data = json.loads(doc)
except:
json_data = {}
if json_data.has_key("body"):
if type(json_data["body"]) is types.DictionaryType:
json_data = json_data["body"]
else:
return
if json_data.has_key("nameCn"):
name = json_data["nameCn"]
if json_data.has_key("nameCn"):
name = json_data["nameCn"]
if json_data.has_key("picCollections"):
poster_dict = json_data["picCollections"]
if type(poster_dict) is types.DictionaryType:
if poster_dict.has_key('400*300'):
poster = poster_dict['400*300']
if poster == "":
for each in poster_dict:
if poster_dict[each] != "":
poster = poster_dict[each]
if self.images.has_key(pid):
poster = self.images[pid]
if json_data.has_key("score"):
try:
point = float(json_data["score"])
except:
point = 0.0
if json_data.has_key("releaseDate"):
shootYear_P = json_data["releaseDate"]
if re.search(r'(\d{4})-\d{2}-\d{2}', shootYear_P):
shootYear = re.search(r'(\d{4})-\d{2}-\d{2}',
shootYear_P).group(1)
elif re.search(r'^\d{4}$', shootYear_P):
shootYear = shootYear_P
if json_data.has_key("directory"):
director_P = json_data["directory"]
if type(director_P) is types.UnicodeType:
director = director_P.strip().replace(" ", ",")
if json_data.has_key("starring"):
star_P = json_data["starring"]
if type(star_P) is types.UnicodeType:
star = star_P.strip().replace(" ", ",")
if json_data.has_key("area"):
area_P = json_data["area"]
if type(area_P) is types.UnicodeType:
area = area_P.strip().replace(" ", ",")
if json_data.has_key("subCategory"):
ctype_P = json_data["subCategory"]
if type(ctype_P) is types.UnicodeType:
ctype = ctype_P.strip().replace(" ", ",")
if json_data.has_key('description'):
intro = json_data['description']
if json_data.has_key('language'):
programLanguage = json_data['language']
self.program["name"] = spiderTool.changeName(name)
self.program['poster'] = spiderTool.listStringToJson('url', poster)
self.program['point'] = point
self.program['shootYear'] = shootYear
self.program['star'] = spiderTool.listStringToJson('name', star)
self.program['director'] = spiderTool.listStringToJson(
'name', director)
self.program['ctype'] = spiderTool.listStringToJson('name', ctype)
self.program['programLanguage'] = programLanguage
self.program['area'] = spiderTool.listStringToJson('name', area)
self.program['intro'] = intro
self.program['mainId'] = mainId
seed_sub = "http://static.app.m.letv.com/android/mod/mob/ctl/videolist/act/detail/id/%s/vid/25520328/b/1/s/60/o/-1/m/0/pcode/010110014/version/5.2.3.mindex.html" % (
pid)
self.secondSpider(seed_sub)
