def step2_ebook(self, params):
try:
#"""只适用在QQ阅读部分,获取评论的url列表"""
bid = params.customized['bid']
jsoncontent = json.loads(params.content)
if not jsoncontent.has_key('data'):
Logger.log(params.originalurl,
constant.ERRORCODE_SITE_NOGET_COMMNETS)
return
comments_count = jsoncontent['data']['total']
page_count = jsoncontent['data']['pageCount']
# 判断增量
cmtnum = CMTStorage.getcount(params.originalurl, True)
NewsStorage.setcmtnum(params.originalurl, comments_count)
if cmtnum >= comments_count:
return
# 判断10页
if int(page_count) >= self.maxpages:
page_count = self.maxpages
for page in range(1, page_count + 1, 1):
commentinfo_url = self.EBOOK_COMMENTS_URL.format(site='intro',
bid=bid,
page=page)
self.storeurl(commentinfo_url, params.originalurl,
self.STEP_COMMENT_NEXT_PAGE)
except Exception, e:
Logger.printexception()
def step3_ebook(self, params):
try:
jsoncontent = json.loads(params.content)
if not jsoncontent.has_key('data'):
return
html = jsoncontent['data']['listHtml']
if not html:
return
soup = BeautifulSoup(html, 'lxml')
divs = soup.select('div.cf')
if not divs:
return
for div in divs:
# commentList > dl:nth-child(1) > div.cf > dd > p:nth-child(2)
content = div.select('dd > p')[1].get_text()
curtime = TimeUtility.getuniformtime(
div.select('dd > p')[0].get_text().split('|')[-1])
nick = div.select('dd > p')[0].get_text().split('|')[0]
if not CMTStorage.exist(params.originalurl, content, curtime,
nick):
CMTStorage.storecmt(params.originalurl, content, curtime,
nick)
except Exception, e:
Logger.printexception()
def process(self, params):
Logger.getlogging().info(params.url)
try:
if params.step is KanKanComments.STEP_1:
# 获取播放量（不是所有的视频都有播放量）
self.setclicknum(params)
if self.r.match(self.TYPE1, params.originalurl):
# Step1: 通过原始url得到moveid，得到获取评论的首页url。
movieid = self.r.parse(self.TYPE1, params.url)[0]
Logger.getlogging().debug(movieid)
commentinfo_url = KanKanComments.COMMENTS_URL1.format(
movieid=movieid, page=1, perpage=self.PERPAGE)
self.storeurl(commentinfo_url, params.originalurl,
KanKanComments.STEP_2, {'movieid': movieid})
elif self.r.match(self.TYPE2, params.originalurl):
# Step1: 通过原始url得到type和sid,得到获取评论的首页url
self.substep1(params, self.TYPE2)
elif self.r.match(self.TYPE3, params.originalurl):
# Step1: 通过原始url得到type和sid,vchannel得到获取评论的首页url
self.substep1(params, self.TYPE3)
elif params.step == KanKanComments.STEP_2:
self.step2(params)
elif params.step == KanKanComments.STEP_3:
self.step3(params)
elif params.step == KanKanComments.STEP_CLICK:
self.step_click(params)
except:
Logger.printexception()
def ifengnews_step2(self, params):
try:
oriurl = params.customized['oriurl']
jsoncontent = json.loads(params.content)
clicknum = float(jsoncontent.get('join_count', '-1'))
if clicknum > 0:
NewsStorage.setclicknum(params.originalurl, clicknum)
curcmtnum = float(jsoncontent['count'])
NewsStorage.setcmtnum(params.originalurl, curcmtnum)
dbcmtnum = CMTStorage.getcount(params.originalurl, True)
if dbcmtnum >= curcmtnum:
return
# 循环取得评论的url
pages = int(math.ceil(
float(curcmtnum - dbcmtnum) / self.page_size))
if pages >= self.maxpages:
pages = self.maxpages
# 拼出第一页之外的其他所有评论url
for index in range(1, pages + 1, 1):
if index == 1:
self.ifengnews_step3(params)
continue
commentinfo_url = IfengNewsComments.COMMENTS_URL.format(
oriurl=oriurl, pg=index, ps=self.page_size)
self.storeurl(commentinfo_url, params.originalurl,
IfengNewsComments.IFENG_NEWS_NEXT_PAGE)
except:
Logger.printexception()
def get(self, url):
saveJson = {}
try:
Logger.getlogging().debug('Downloading: {url}'.format(url=url))
request = urllib2.Request(url, headers=self.headers)
response = urllib2.urlopen(request, timeout=self.timeout)
code = response.getcode()
info = response.info()
# 判断返回的code，如果不是200，则返回空
if code == 200:
html = response.read()
if (("Content-Encoding" in info) and (info['Content-Encoding'] == "gzip")):
html = zlib.decompress(html, 16 + zlib.MAX_WBITS);
Logger.getlogging().debug('Request Sucessed: {url}'.format(url=url))
else:
Logger.getlogging().error('open {url} error, code = {code}'.format(url=url, code=code))
Logger.getlogging().error('Request Failed: {url}'.format(url=url))
return None
except:
Logger.getlogging().error('Request Failed: {url}'.format(url=url))
Logger.printexception()
return None
charset = RegexUtility.getid('charset', html)
html = Common.trydecode(html, charset)
saveJson['foundin'] = Common.urlenc(url)
saveJson['html'] = Common.urlenc(html.encode(constant.CHARSET_UTF8))
saveJson['crawler_time'] = int(time.time())
jsonStr = json.dumps(saveJson)
return jsonStr
def step2(self, params):
try:
Logger.getlogging().info("xinhuaComments.STEP_2")
# 将STEP_1中的commentinfo_url传下来
newsId = params.customized['newsId']
comments_info = json.loads(params.content)
comments_count = comments_info['totalRows']
NewsStorage.setcmtnum(params.originalurl, comments_count)
page_count = comments_info['totalPage']
# 判断增量
cmtnum = CMTStorage.getcount(params.originalurl, True)
if cmtnum >= comments_count:
return
# 判断增量
if page_count >= self.maxpages:
page_count = self.maxpages
for index in range(0, int(page_count)):
commentinfo_url = xinhuaNewsComments.COMMENTS_URL_NEWS.format(
newsId=newsId, pid=(index + 1))
self.storeurl(commentinfo_url, params.originalurl,
xinhuaNewsComments.STEP_3)
except:
Logger.printexception()
def step3bbs(self, params):
Logger.getlogging().info("JoyComments.STEP_3")
# Step3: 通过Step2设置的url，得到所有评论，抽取评论
try:
commentsinfo = json.loads(params.content)
commentsinfo['result']['mainreplys']['rows']
except:
Logger.getlogging().warning(
'{url} Errorcode:40000'.format(url=params.originalurl))
Logger.printexception()
return
# 获取评论
for index in range(
0, int(len(commentsinfo['result']['mainreplys']['rows'])), 1):
# 提取时间
# cmti = CommentInfo()
content = commentsinfo['result']['mainreplys']['rows'][index][
'reply']['reply']['body']['text']
curtime = TimeUtility.getuniformtime(
str(commentsinfo['result']['mainreplys']['rows'][index]
['reply']['reply']['post_time']))
nick = commentsinfo['result']['mainreplys']['rows'][index][
'reply']['user']['name']
if not CMTStorage.exist(params.originalurl, content, curtime,
nick):
CMTStorage.storecmt(params.originalurl, content, curtime, nick)
def step1(self, params):
try:
url = params.originalurl
videoId = params.customized['videoId']
params.content = params.content[params.content.index('{'):params.content.rindex('}')+1]
jsonData = json.loads(params.content)['data']
hasCmts = jsonData['page']['count']
# 是否有评论
if not hasCmts:
return
# 比较增量：获取现在评论数目及上次采集的评论数目
currCmtsCount = jsonData['page']['acount']
NewsStorage.setcmtnum(url, currCmtsCount)
prevCmtsCount = int(CMTStorage.getcount(url))
# 若没有评论更新，跳过
if prevCmtsCount >= currCmtsCount:
return
# 更新评论数
pageNum = int(math.ceil((hasCmts-prevCmtsCount)/self.pageSize))
# 上次采集数据的截止时间
# 生成分页url并传递给共通模块
for page in range(1, pageNum + 1):
if page == 1:
self.step2(params)
pageUrl = self.pageUrl.format(page = page, videoId = videoId)
self.storeurl(pageUrl, url, self.STEP_CMTS)
except:
Logger.printexception()
def analysis(self, line, method):
try:
js = json.loads(line)
param = ProcessParam()
param.crawler_time = TimeUtility.getuniformtime(js['crawler_time'])
param.url = Common.urldec(js['foundin'])
param.content = js['html']
if method == constant.REQUEST_TYPE_POST:
param.data = js['data']
if js['html'][:3] == constant.GZIP_CODE:
param.content = zlib.decompress(param.content,
16 + zlib.MAX_WBITS)
# decode
content = Common.urldec(param.content)
charset = RegexUtility.getid('charset', content)
content = Common.trydecode(content, charset)
param.content = content
if 'property' in js:
for property in js['property']:
if not property.has_key('result'):
continue
if property['property_name'] == u'page_body':
param.page_body = Common.trydecode(
Common.urldec(property['result'][0]['text']),
constant.CHARSET_GBK)
elif property['property_name'] == u'page_title':
param.page_title = Common.trydecode(
Common.urldec(property['result'][0]['text']),
constant.CHARSET_GBK)
elif property['property_name'] == u'html_time':
param.html_time = TimeUtility.getuniformtime(
property['result'][0]['text'])
return param
except:
Logger.printexception()
def process(self, proparam):
Logger.getlogging().info(proparam.url)
# if NewsStorage.getclicknum(proparam.originalurl) <= 0:
# contentid = self.r.parse('^http://www\.kumi\.cn/(\w+)/(\d+)(_\d)?\.html.*', proparam.originalurl)[1]
# clickurl = self.VIDEO_CLICKURL.format(contentid=contentid)
# self.storeurl(clickurl, proparam.originalurl, self.STEP_TVCLICK)
try:
if self.r.search('^http://.*\.kumi\.cn/.*', proparam.originalurl):
contentid = self.r.parse(
'^http://www\.kumi\.cn/(\w+)/(\d+)(_\d)?\.html.*',
proparam.originalurl)[1]
clickurl = self.VIDEO_CLICKURL.format(contentid=contentid)
self.storeurl(clickurl, proparam.originalurl,
self.STEP_TVCLICK)
if 'donghua' in proparam.originalurl:
topicSourceId = re.findall(
r'^http://www\.kumi\.cn/donghua/(\d+)(_\d)?\.html',
proparam.originalurl).__getitem__(0).__getitem__(0)
else:
topicSourceId = re.findall(
r'^http://xiaoyouxi\.kumi\.cn/(\d+)(_\d)?\.htm',
proparam.originalurl).__getitem__(0).__getitem__(0)
self.createobject().getcomments(proparam, topicSourceId, 3, 2)
elif proparam.setp == self.STEP_TVCLICK:
self.setclicknum_tv(proparam)
except:
Logger.printexception()
def delete(self, table, where, relation='and', retrycount=0):
if self.check():
if where and isinstance(where, dict):
tempwhere = [
' {key}=\"{value}\" '.format(key=key, value=value)
for key, value in where.iteritems()
]
tempwhere = 'and'.join(tempwhere)
sql = SQLDAO.DELETETABLE.format(table=table, where=tempwhere)
else:
sql = SQLDAO.DELETETABLE.format(table=table, where=where)
Logger.getlogging().debug(sql)
try:
cur = self.connect.cursor()
cur.execute(sql)
cur.close()
self.connect.commit()
return True
except:
if retrycount == SQLDAO.MAX_RETRY_TIMES:
Logger.getlogging().error(sql)
Logger.printexception()
return False
retrycount += 1
return self.delete(table, where, relation, retrycount)
def insert(self, table, keys, values, mutli=False, retrycount=0):
if self.check():
sql = SQLDAO.INSERTTABLE.format(table=table,
keys=', '.join(keys),
values=', '.join(['%s'] *
len(keys)))
try:
cur = self.connect.cursor()
if not mutli:
tempvalues = [
'\"{value}\"'.format(value=item) for item in values
]
Logger.getlogging().debug(sql % tuple(tempvalues))
cur.execute(sql, tuple(values))
else:
tempvalues = []
for value in values:
tempvalue = [
'\"{value}\"'.format(value=item) for item in value
]
Logger.getlogging().debug(sql % tuple(tempvalue))
tempvalues.append(tuple(value))
cur.executemany(sql, tempvalues)
cur.close()
self.connect.commit()
return True
except:
if retrycount == SQLDAO.MAX_RETRY_TIMES:
Logger.getlogging().error(sql)
Logger.printexception()
return False
retrycount += 1
return self.insert(table, keys, values, mutli, retrycount)
def setclick(self, params):
try:
content = json.loads(params.content)
# content=[播放量，评论，X，X,弹幕,收藏数，投焦数，X]
cmtnum = content[1]
clicknum = content[0]
votenum = content[-2]
fansnum = content[-3]
if not cmtnum:
cmtnum = 0
if not clicknum:
clicknum = 0
if not votenum:
votenum = 0
if not fansnum:
fansnum = 0
NewsStorage.seturlinfo(params.originalurl,
data={
SQLDAO.SPIDER_TABLE_NEWS_CMTNUM: cmtnum,
SQLDAO.SPIDER_TABLE_NEWS_CLICKNUM:
clicknum,
SQLDAO.SPIDER_TABLE_NEWS_VOTENUM:
votenum,
SQLDAO.SPIDER_TABLE_NEWS_FANSNUM:
fansnum
})
except:
Logger.printexception()
def step3(self, params):
"""通过评论的url获取评论"""
#相对之前的版本，本次更新变动：
#comments存储的接口为CMTStorage.storecmt(),参数为originalurl, 评论内容, 评论发布时间, 用户
#存储的内容增加了 评论发布时间, 用户
try:
jsondata = json.loads(params.content)
if jsondata['comments']:
for comment in jsondata['comments']:
content = comment['content']
curtime = TimeUtility.getuniformtime(
comment['create_time'])
nick = comment['passport']['nickname']
if not CMTStorage.exist(params.originalurl, content,
curtime, nick):
CMTStorage.storecmt(params.originalurl, content,
curtime, nick)
reply = comment['comments']
while reply:
for comment in comment['comments']:
content = comment['content']
curtime = TimeUtility.getuniformtime(
comment['create_time'])
nick = comment['passport'].get(
'nickname', 'anonymous')
if not CMTStorage.exist(params.originalurl,
content, curtime, nick):
CMTStorage.storecmt(params.originalurl,
content, curtime, nick)
reply = comment['comments']
except:
Logger.printexception()
Logger.log(params.originalurl, constant.ERRORCODE_SITE_NOGET_SITE)
def baidutiebasearch_step3(self, params):
soup = BeautifulSoup(params.content, 'html5lib')
post_list = soup.select('.s_post_list > .s_post')
urllist = []
for item in post_list:
try:
title = item.select_one('.p_title > a').get_text().strip()
href = item.select_one('.p_title > a').get('href')
pubtimeobj = item.find(attrs={'class':'p_green p_date'})
if not pubtimeobj:
Logger.log(params.url, constant.ERRORCODE_SITE_NOGET_XPATHVALUE)
continue
pubtime = pubtimeobj.get_text()
pubtime = getuniformtime(pubtime)
Logger.getlogging().debug(title)
Logger.getlogging().debug(pubtime)
if self.isyestoday(pubtime):
Logger.getlogging().debug('https://tieba.baidu.com'+href)
urllist.append('https://tieba.baidu.com'+href)
else:
Logger.log(params.url, constant.ERRORCODE_WARNNING_NOMATCHTIME)
except:
Logger.printexception()
if len(urllist) > 0:
self.__storeurllist__(urllist, SPIDER_S2_WEBSITE_TIEBA)
def step2(self, params):
""""""
try:
key = params.customized['key']
soup = BeautifulSoup(params.content, 'html5lib')
#print soup
#searchListOne = soup.select('.searchListOne > ul')
searchListOne = soup.select('.searchListOne > ul > li > div')
if not searchListOne:
Logger.getlogging().warning('{}:40000 No urllist'.format(
params.originalurl))
return
lis = soup.select(
'.searchListOne > ul > li'
)[:-1] #最后一个<li id=search_msg style="display:none"></li>，过滤掉
urllist = []
for li in lis:
url = li.select_one('h3 > a').get('href')
#print '*********',url
tm = li.select('.source > span')[0].get_text()
tm = getuniformtime(tm)
now = getuniformtime(str(time.time()))
cmt_num = li.select('.source > span')[-1].get_text()
title = li.select_one('h3').get_text()
if Common.checktitle(Common.urldec(key), title):
if compareNow(tm, self.querylastdays):
urllist.append(url)
if len(urllist) > 0:
self.__storeurllist__(urllist, SPIDER_S2_WEBSITE_TIEBA)
except:
#traceback.print_exc()
Logger.printexception()
Logger.getlogging().error(
'extract comment error from {site}'.format(site=params.url))
def step2(self, params):
"""获取评论的url"""
try:
newsId = params.customized['newsId']
jsondata = json.loads(params.content)
backflag = False
if jsondata:
comments = []
for comment in jsondata:
cmti = CommentInfo()
if URLStorage.storeupdatetime(params.originalurl,
str(comment['commentTime'])):
cmti.content = comment['commentContent']
cmti.commentid = comment['commentId']
comments.append(cmti)
else:
backflag = True
self.commentstorage.store(params.originalurl, comments)
if backflag == False:
self.commentstorage.store(params.originalurl, comments)
self.pageno += 1
comment_url = self.COMMENTS_URL.format(
self.pageno, self.page_size, newsId)
self.storeurl(comment_url, params.originalurl,
self.STEP_COMMENT_FIRST_PAGE,
{'newsId': newsId})
except:
Logger.printexception()
def scanning():
whoami = SpiderConfigure.getconfig(const.SPIDER_LOCAL_DOMAIN, const.SPIDER_LOCAL_WHOAMI)
scanningPath = SpiderConfigure.getconfig(const.SPIDER_LOCAL_DOMAIN, whoami + constant.DOWNLOADER_URL_PATH)
donepath = SpiderConfigure.getconfig(const.SPIDER_LOCAL_DOMAIN, whoami + constant.DOWNLOADER_DONE_PATH)
FileUtility.removefiles(donepath)
backupPath = os.path.join(SpiderConfigure.getconfig(const.SPIDER_LOCAL_DOMAIN, const.DOWNLOADER_URL_BACKUP), TimeUtility.getcurrentdate())
interval = SpiderConfigure.getconfig(const.SPIDER_LOCAL_DOMAIN, const.DOWNLOADER_INTERVAL)
FileUtility.mkdirs(scanningPath)
FileUtility.mkdirs(backupPath)
while True:
Logger.getlogging().debug('scanning')
flag = False
for filename in os.listdir(scanningPath):
try:
urlfilepath = os.path.join(scanningPath, filename)
backupfile = os.path.join(backupPath, filename)
if os.path.isfile(urlfilepath) and 'tmp' not in filename:
Logger.getlogging().info('Get url file:{file}'.format(file=filename))
FileUtility.copy(urlfilepath, backupfile)
download(urlfilepath)
if not flag:
flag = True
except:
Logger.printexception()
if not flag:
Logger.getlogging().debug('scanning interval sleeping {interval}s'.format(interval=interval))
time.sleep(int(interval))
def step2(self,params):
soup = BeautifulSoup(params.content, 'html5lib')
if soup.find(attrs={"id":re.compile('noresult_part._container')}):
Logger.getlogging().warning('{url}:40000 No results'.format(url=params.url))
return
results = soup.select('.results > .vrwrap')
if not results:
Logger.getlogging().warning('{url}:40000 No results'.format(url=params.url))
return
urllist = []
newurllist = []
for item in results:
try:
if not item.select_one('h3.vrTitle > a'):
continue
title = item.select_one('h3.vrTitle > a').get_text()
href = item.select_one('h3.vrTitle > a').get('href')
timestr = item.select_one('.news-detail > .news-info > .news-from').get_text()
times = getuniformtime(timestr)
Logger.getlogging().debug('title:'+ title)
Logger.getlogging().debug('time:'+ times)
if compareNow(times, self.querylastdays):
Logger.getlogging().debug('href:'+ href)
urllist.append(href)
newitem = item.select_one('#news_similar')
if newitem:
newhref = 'http://news.sogou.com/news'+newitem.get('href')
Logger.getlogging().debug('newhref:'+ newhref)
newurllist.append(newhref)
except:
Logger.printexception()
if len(urllist) > 0:
self.__storeurllist__(urllist, SPIDER_S2_WEBSITE_NEWS)
if len(newurllist) > 0:
self.__storeqeuryurllist__(newurllist, self.NEWS_EACH_2)
def process(self, params):
try:
if params.step is None:
# 从html源码中获取拼接评论url的参数
operaId = self.r.getid('operaId', params.content, '\s*:\s*')
contentId = self.r.getid('contentId', params.content,
'\s*:\s*')
# 拼接第一页评论url
comments_url = Comments.COMMENTS_URL % (operaId, contentId, 1,
Comments.PAGE_SIZE)
#通知下载平台，根据评论url获取第一页评论内容
self.storeurl(comments_url, params.originalurl,
Comments.STEP_2, {
'operaId': operaId,
'contentId': contentId
})
#获取第一页评论内容，循环获取全部评论url
elif params.step == Comments.STEP_2:
self.step2(params)
#解析评论数据
elif params.step == Comments.STEP_3:
self.step3(params)
except:
Logger.printexception()
def step2(self, params):
try:
Logger.getlogging().info("Kr36Comments.STEP_2")
# 将STEP_1中的cid传下来
cid = params.customized['cid']
jsoncontent = json.loads(params.content)
comments_count = jsoncontent['data']['total_items']
page_count = jsoncontent['data']['total_pages']
# 判断增量
cmtnum = CMTStorage.getcount(params.originalurl)
if cmtnum >= comments_count:
return
#最多只取十页评论
# page_num = int(math.ceil(float(comments_count - cmtnum) / self.page_size))
if page_count >= self.maxpages:
page_count = self.maxpages
lasttime = CMTStorage.getlastpublish(params.originalurl,True)
for page in range(1, page_count+1, 1):
commentinfo_url = Kr36Comments.COMMENT_URL.format(cid, self.page_size, page)
self.storeurl(commentinfo_url, params.originalurl, Kr36Comments.STEP_3,lasttime)
except:
Logger.printexception()
def readFile(urlpath, filename):
whoami = SpiderConfigure.getconfig(const.SPIDER_POST_DOMAIN,
const.SPIDER_POST_WHOAMI)
donepath = SpiderConfigure.getconfig(
const.SPIDER_POST_DOMAIN, whoami + constant.DOWNLOADER_DONE_PATH)
FileUtility.mkdirs(donepath)
writeTmpfile = donepath + filename + '.tmp'
now = str(time.time()).split('.')[0]
writefile = donepath + filename + '.txt.' + now + '.done'
if os.path.exists(writeTmpfile):
os.remove(writeTmpfile)
if os.path.exists(writefile):
os.remove(writefile)
Logger.getlogging().debug('post_done start:{f}'.format(f=writefile))
with open(urlpath, 'r') as fp:
lines = fp.readlines()
os.mknod(writeTmpfile)
for line in lines:
jsonLine = json.loads(line)
try:
jsonStr = downPost(jsonLine)
with open(writeTmpfile, 'a+') as filetemp:
filetemp.write(jsonStr + '\n')
Logger.getlogging().debug(
'{url}:Post request sucessed'.format(url=jsonLine['url']))
except:
Logger.getlogging().warning(
'{url}:Post request failed'.format(url=jsonLine['url']))
Logger.printexception()
if os.path.exists(writeTmpfile):
os.rename(writeTmpfile, writefile)
Logger.getlogging().debug('post_done end:{f}'.format(f=writefile))
FileUtility.remove(urlpath)
def step1(self, params):
"""获取评论的首页url"""
try:
#获取上一次的最新更新时间
before_update = CMTStorage.getlastpublish(params.originalurl)
#获取其他信息，拼接url
url_id = None
if self.r.search('^http[s]{0,1}://v\.qq\.com/.*',
params.originalurl):
#{"comment_id":"1167760750","result":{"code":0,"msg":"Success!","ret":0},"srcid":"c0016r7fo07","srcid_type":1001}
url_id = self.r.getid('comment_id', params.content)
else:
url_id = self.r.getid('cmt_id', params.content)
if not url_id:
url_id = self.r.getid('aid', params.content)
if not url_id:
url_id = self.r.getid('commId', params.content)
if url_id:
comment_url = self.COMMENTS_URL.format(url_id, 0,
self.page_size)
self.storeurl(
comment_url, params.originalurl,
self.STEP_COMMENT_NEXT_PAGE, {
'url_id': url_id,
'comment_id': 0,
'before_update': before_update
})
except:
Logger.printexception()
def step2_ifeng_xiaobg(self, params):
try:
jsoncontent = json.loads(params.content)
clicknum = float(jsoncontent.get('join_count', '-1'))
if clicknum > 0:
NewsStorage.setclicknum(params.originalurl, clicknum)
curcmtnum = jsoncontent['count']
NewsStorage.setcmtnum(params.originalurl, curcmtnum)
dbcmtnum = CMTStorage.getcount(params.originalurl, True)
if dbcmtnum >= curcmtnum:
return
# 循环取得评论的url
pages = int(math.ceil(
float(curcmtnum - dbcmtnum) / self.page_size))
if pages >= self.maxpages:
pages = self.maxpages
for index in range(1, pages + 1, 1):
if index == 1:
self.ifengnews_step3(params)
continue
self.post_data['p'] = index
self.storeposturl(self.post_url, params.originalurl,
self.IFENG_NEWS_NEXT_PAGE,
IfengNewsComments.post_data)
except:
Logger.printexception()
def gets2url(self, params):
# 获取文本
contents = json.loads(params.content)
query = Common.urldec(params.customized['query'])
urllist = []
for item in contents['video_list']:
try:
vid = item['vid']
if item.get('categoryName', '') == u"体育":
url = 'http://sports.le.com/video/{vid}.html'.format(
vid=vid)
else:
url = 'http://www.le.com/ptv/vplay/{vid}.html'.format(
vid=vid)
curtime = item['ctime']
#print TimeUtility.getuniformtime(curtime)
title = item['name']
if self.compareNow(curtime):
if self.checktitle(query, title):
#Logger.getlogging().info(title)
urllist.append(url)
else:
Logger.log(url,
constant.ERRORCODE_WARNNING_NOMATCHTITLE)
else:
Logger.log(url, constant.ERRORCODE_WARNNING_NOMATCHTIME)
except:
Logger.printexception()
# 获取最终url列表
if len(urllist) > 0:
self.__storeurllist__(urllist, SPIDER_S2_WEBSITE_VIDEO)
def step2(self, params):
"""获取评论的其他url"""
try:
comments = json.loads(params.content)
topic_id = comments['topic_id']
curcmtnum = float(comments.get('cmt_sum', -1))
#clicknum = float(comments.get('participation_sum',-1))
NewsStorage.setcmtnum(params.originalurl, curcmtnum)
#NewsStorage.setclicknum(params.originalurl, clicknum)
dbcmtnum = CMTStorage.getcount(params.originalurl, True)
if dbcmtnum >= curcmtnum:
return
page_num = int(
math.ceil(float(curcmtnum - dbcmtnum) / self.page_size))
if page_num >= self.maxpages:
page_num = self.maxpages
for page in range(1, page_num + 1):
if self.r.search('http[s]{0,1}://.*tv\.sohu.com/.*',
params.originalurl):
url = self.COMMENTS_URL.format(self.tv_client_id, topic_id,
page, self.tv_page_size)
else:
url = self.COMMENTS_URL.format(self.client_id, topic_id,
page, self.page_size)
self.storeurl(url, params.originalurl,
self.STEP_COMMENT_NEXT_PAGE)
except:
Logger.printexception()
Logger.log(params.originalurl, constant.ERRORCODE_SITE_NOGET_SITE)
def step2(self, params):
keyword = params.customized['keyword']
query = Common.urldec(keyword)
jsondata = json.loads(params.content)
# 获取分页数
html = jsondata['html']
soup = bs(html, 'html5lib')
videoUrlList = []
videoList = soup.select('li.video')
for video in videoList:
try:
videoUrl = 'https:' + video.select_one('a').get('href')
videoUrl = videoUrl.split('?')[0] + '/'
title = video.select_one('a').get('title')
pubtime = video.find(attrs={
'class': 'so-icon time'
}).get_text().strip()
if self.compareNow(TimeUtility.getuniformtime(pubtime)):
if self.checktitle(query, title):
videoUrlList.append(videoUrl)
self.__storeurl__(videoUrl, pubtime,
SPIDER_S2_WEBSITE_VIDEO)
else:
Logger.log(videoUrl,
constant.ERRORCODE_WARNNING_NOMATCHTITLE)
else:
Logger.log(videoUrl,
constant.ERRORCODE_WARNNING_NOMATCHTIME)
except:
Logger.printexception()
def process(self, proparam):
Logger.getlogging().info(proparam.url)
try:
if proparam.step is LeiphoneComments.STEP_1:
# 取得url中的id
articleId = self.r.getid('data-article_id', proparam.content)
comments_url = LeiphoneComments.COMMENTS_URL % (articleId)
self.storeurl(comments_url, proparam.originalurl,
LeiphoneComments.STEP_2,
{'articleId': articleId})
elif proparam.step == LeiphoneComments.STEP_2:
articleId = proparam.customized['articleId']
comments = proparam.content[
proparam.content.index('{'):proparam.content.rindex('}') +
1]
comments = json.loads(comments)
comments_count = float(comments['allCount']['num'])
NewsStorage.setcmtnum(proparam.originalurl, comments_count)
# 取得评论件数
if int(comments_count) == 0:
return
# 判断增量
cmtnum = CMTStorage.getcount(proparam.originalurl, True)
if cmtnum >= comments_count:
return
# 取得评论
self.geturlcomments(proparam)
# 取得评论url
# comments_url = LeiphoneComments.COMMENTS_URL % (articleId)
# self.storeurl(comments_url, proparam.originalurl, LeiphoneComments.STEP_3)
elif proparam.step == LeiphoneComments.STEP_3:
return
# # 取得评论的正则表达式
# comments = re.findall(r'content":"(.+?)","paragraph_id"', proparam.content)
# commentsInfo = []
# commentsTime = self.r.parse(r'origin_created":"(\d+)","member_avatarPath"', proparam.content)
# # 取得评论
# index = 0
# for comment in comments:
# comment = eval('u"' + comment + '"')
# cmti = CommentInfo()
# cmti.content = comment.encode('utf-8')
# if URLStorage.storeupdatetime(proparam.originalurl, getuniformtime(commentsTime[index])):
# commentsInfo.append(cmti)
# index = index + 1
#
# # 保存获取的评论
# if len(commentsInfo) > 0:
# self.commentstorage.store(proparam.originalurl, commentsInfo)
else:
return
except:
Logger.printexception()
def set_click(self, params):
try:
vid = params.customized['vid']
data = json.loads(params.content)
clicknum = data['data'][vid]
NewsStorage.setclicknum(params.originalurl, clicknum)
except:
Logger.printexception()
def process(self, params):
try:
if params.step == self.STEP_PAGES:
self.step1(params)
elif params.step == self.STEP_VIDEOS:
self.step2(params)
except:
Logger.printexception()