def step3(self, params):
"""通过评论的url获取评论"""
#相对之前的版本，本次更新变动：
#comments存储的接口为CMTStorage.storecmt(),参数为originalurl, 评论内容, 评论发布时间, 用户
#存储的内容增加了 评论发布时间, 用户
try:
jsondata = json.loads(params.content)
if jsondata['comments']:
for comment in jsondata['comments']:
content = comment['content']
curtime = TimeUtility.getuniformtime(
comment['create_time'])
nick = comment['passport']['nickname']
if not CMTStorage.exist(params.originalurl, content,
curtime, nick):
CMTStorage.storecmt(params.originalurl, content,
curtime, nick)
reply = comment['comments']
while reply:
for comment in comment['comments']:
content = comment['content']
curtime = TimeUtility.getuniformtime(
comment['create_time'])
nick = comment['passport'].get(
'nickname', 'anonymous')
if not CMTStorage.exist(params.originalurl,
content, curtime, nick):
CMTStorage.storecmt(params.originalurl,
content, curtime, nick)
reply = comment['comments']
except:
Logger.printexception()
Logger.log(params.originalurl, constant.ERRORCODE_SITE_NOGET_SITE)
def analysis(self, line, method):
try:
js = json.loads(line)
param = ProcessParam()
param.crawler_time = TimeUtility.getuniformtime(js['crawler_time'])
param.url = Common.urldec(js['foundin'])
param.content = js['html']
if method == constant.REQUEST_TYPE_POST:
param.data = js['data']
if js['html'][:3] == constant.GZIP_CODE:
param.content = zlib.decompress(param.content,
16 + zlib.MAX_WBITS)
# decode
content = Common.urldec(param.content)
charset = RegexUtility.getid('charset', content)
content = Common.trydecode(content, charset)
param.content = content
if 'property' in js:
for property in js['property']:
if not property.has_key('result'):
continue
if property['property_name'] == u'page_body':
param.page_body = Common.trydecode(
Common.urldec(property['result'][0]['text']),
constant.CHARSET_GBK)
elif property['property_name'] == u'page_title':
param.page_title = Common.trydecode(
Common.urldec(property['result'][0]['text']),
constant.CHARSET_GBK)
elif property['property_name'] == u'html_time':
param.html_time = TimeUtility.getuniformtime(
property['result'][0]['text'])
return param
except:
Logger.printexception()
def seturlinfos(params):
id = NewsStorage.getid(params.url)
if NewsStorage.exist(params.url):
doc = NewsStorage.getdoc(params.url)
data = {}
#data[SQLDAO.SPIDER_TABLE_NEWS_TYPE] = params.type
data[SQLDAO.SPIDER_TABLE_NEWS_TITLE] = Common.strfilter(
params.title)
if params.type != constant.SPIDER_S2_WEBSITE_VIDEO:
data[SQLDAO.SPIDER_TABLE_NEWS_BODY] = Common.strfilter(
params.body)
if doc.get(SQLDAO.SPIDER_TABLE_NEWS_PUBLISH_DATE,
TimeUtility.getintformtime(
0)) == TimeUtility.getintformtime(0):
data[
SQLDAO.
SPIDER_TABLE_NEWS_PUBLISH_DATE] = TimeUtility.getuniformtime(
params.pubtime)
data[SQLDAO.SPIDER_TABLE_NEWS_CMTNUM] = params.cmtnum
data[SQLDAO.SPIDER_TABLE_NEWS_CLICKNUM] = params.clicknum
data[SQLDAO.SPIDER_TABLE_NEWS_FANSNUM] = params.fansnum
data[SQLDAO.SPIDER_TABLE_NEWS_VOTENUM] = params.votenum
data[SQLDAO.SPIDER_TABLE_NEWS_UPDATE_DATE] = SQLDAO.gettime()
SQLDAO.getinstance().update(SQLDAO.SPIDER_TABLE_NEWS,
{SQLDAO.SPIDER_TABLE_NEWS_ID: id},
data)
else:
data = {}
data[SQLDAO.SPIDER_TABLE_NEWS_TYPE] = params.type
data[SQLDAO.SPIDER_TABLE_NEWS_TITLE] = Common.strfilter(
params.title)
if params.type != constant.SPIDER_S2_WEBSITE_VIDEO:
data[SQLDAO.SPIDER_TABLE_NEWS_BODY] = Common.strfilter(
params.body)
data[SQLDAO.
SPIDER_TABLE_NEWS_PUBLISH_DATE] = TimeUtility.getuniformtime(
params.pubtime)
data[SQLDAO.SPIDER_TABLE_NEWS_CMTNUM] = params.cmtnum
data[SQLDAO.SPIDER_TABLE_NEWS_CLICKNUM] = params.clicknum
data[SQLDAO.SPIDER_TABLE_NEWS_FANSNUM] = params.fansnum
data[SQLDAO.SPIDER_TABLE_NEWS_VOTENUM] = params.votenum
data[SQLDAO.SPIDER_TABLE_NEWS_UPDATE_DATE] = SQLDAO.gettime()
data[SQLDAO.SPIDER_TABLE_NEWS_ID] = id
data[SQLDAO.SPIDER_TABLE_NEWS_URL] = params.url
data[SQLDAO.SPIDER_TABLE_NEWS_QUERY] = params.query
data[SQLDAO.SPIDER_TABLE_NEWS_CHANNEL] = params.channel
data[SQLDAO.SPIDER_TABLE_NEWS_CREATE_DATE] = params.createtime
data[SQLDAO.
SPIDER_TABLE_NEWS_MACHINEFLAG] = NewsStorage.LOCALMACHINEFLAG
SQLDAO.getinstance().insert(SQLDAO.SPIDER_TABLE_NEWS,
SQLDAO.SPIDER_TABLE_NEWS_KEYS,
SQLDAO.getvaluesfromkeys(data))
def storecmt(url, content, pubdate, user):
content = Common.strfilter(content)
user = Common.strfilter(user)
pubdate = TimeUtility.getuniformtime(pubdate)
if not CMTStorage.exist(url, content, pubdate, user):
Logger.getlogging().debug(
'url:{url}, content:{content}, pubdate:{pubdate}, user:{user}'.
format(url=url, content=content, pubdate=pubdate, user=user))
id = CMTStorage.getid(url, content, pubdate, user)
data = {
SQLDAO.SPIDER_TABLE_COMMENTS_ID:
id,
SQLDAO.SPIDER_TABLE_COMMENTS_URL:
url,
SQLDAO.SPIDER_TABLE_COMMENTS_PUBLISH_DATE:
pubdate,
SQLDAO.SPIDER_TABLE_COMMENTS_USER:
user,
SQLDAO.SPIDER_TABLE_COMMENTS_CONTENT:
content,
SQLDAO.SPIDER_TABLE_COMMENTS_CREATE_DATE:
SpiderConfigure.getinstance().starttime()
}
SQLDAO.getinstance().insert(
SQLDAO.SPIDER_TABLE_COMMENTS,
SQLDAO.SPIDER_TABLE_COMMENTS_KEYS,
SQLDAO.getvaluesfromkeys(data,
SQLDAO.SPIDER_TABLE_COMMENTS_KEYS))
def getid(url, content, pubdate, user):
content = Common.strfilter(content)
user = Common.strfilter(user)
pubdate = TimeUtility.getuniformtime(pubdate)
return Common.md5(
Common.urlenc(url) + Common.urlenc(content) + pubdate +
Common.urlenc(user))
def bbs_step3(self, params):
try:
xparser = XPathUtility(params.content)
page = params.customized['page']
pagecount = params.customized['pagecount']
comments = []
updatetimes = []
nicks = []
contents = xparser.getcomments('//*[@class="read"]')
mid_times = xparser.getlist('//td[@class="authorname"]')
for times in mid_times:
updatetimes.append(self.r.parse(ur'于(\d+-\d+-\d+ \d+:\d+:\d+)留言', times)[0])
nicks.append(self.r.parse(ur'(.*)于', times)[0])
if page == 0:
mid_index = 1
elif page > 0:
mid_index = 0
comments_number = xparser.getnumber('//*[@id="msgsubject"]/font')
if comments_number != 0:
for index in range(mid_index, len(contents), 1):
curtime = TimeUtility.getuniformtime(updatetimes[index])
content = contents[index]
nick = nicks[index].split('于')[0].split('☆')[-1]
if not CMTStorage.exist(params.originalurl, content, curtime, nick):
CMTStorage.storecmt(params.originalurl, content, curtime, nick)
except Exception, e:
traceback.print_exc()
def get_url_id(self, params):
"""只适用在腾讯视频的部分"""
"cid是电视剧\合集\电影,vid单集"
CID_PATTERN = 'https?://v\.qq\.com/x/cover/(\w+).html'
CID_URL = 'https://ncgi.video.qq.com/fcgi-bin/video_comment_id?otype=json&op=3&cid={cid}'
VID_PATTERN1 = 'https?://v\.qq\.com/x/cover/\w+/(\w+).html'
VID_PATTERN2 = 'https?://v\.qq\.com/x/page/(\w+)\.html'
VID_URL = 'https://ncgi.video.qq.com/fcgi-bin/video_comment_id?otype=json&op=3&vid={vid}'
if self.r.search(CID_PATTERN, params.originalurl):
cid = self.r.parse(CID_PATTERN, params.originalurl)[0]
url = CID_URL.format(cid=cid)
self.storeurl(url, params.originalurl,
self.STEP_COMMENT_FIRST_PAGE)
elif self.r.search(VID_PATTERN1, params.originalurl):
vid = self.r.parse(VID_PATTERN1, params.originalurl)[0]
url = VID_URL.format(vid=vid)
self.storeurl(url, params.originalurl,
self.STEP_COMMENT_FIRST_PAGE)
elif self.r.search(VID_PATTERN2, params.originalurl):
vid = self.r.parse(VID_PATTERN2, params.originalurl)[0]
url = VID_URL.format(vid=vid)
self.storeurl(url, params.originalurl,
self.STEP_COMMENT_FIRST_PAGE)
#publish_date
publish_date = self.r.getid('publish_date', params.content, split=':')
if not publish_date:
publish_date = XPathUtility(params.content).getstring(
'//*[@class="video_tags"]/span|//*[@class="date"]|//*[@class="tag_item"]'
)
publish_date = TimeUtility.getuniformtime(publish_date)
if publish_date:
NewsStorage.setpublishdate(params.originalurl, publish_date)
self.setclick(params)
def step2(self, params):
info = Common.urldec(params.customized['info'])
soup = BeautifulSoup(params.content, 'html5lib')
text_divs = soup.select('.s_r_txt')
urllist = []
if text_divs:
for item in text_divs:
title = item.select_one('h3 > a').get_text()
url = item.select_one('h3 > a').get('href')
curtime = item.select('p')[-1].get_text().strip()
try:
if TimeUtility.compareNow(
TimeUtility.getuniformtime(curtime),
self.querylastdays):
if Common.checktitle(info, title):
urllist.append(url)
else:
Logger.log(
url, constant.ERRORCODE_WARNNING_NOMATCHTITLE)
else:
Logger.log(url,
constant.ERRORCODE_WARNNING_NOMATCHTIME)
except:
urllist.append(url)
self.__storeurllist__(urllist, SPIDER_S2_WEBSITE_VIDEO)
def getclick(self, params):
pattern = 'https?://\w+\.le\.com.*/\w+/(\d+)\.html'
if re.search(pattern, params.originalurl):
if self.r.search(pattern, params.originalurl):
vid = self.r.parse(pattern, params.originalurl)[0]
playcount_url = self.PALYCOUNT_URL.format(vid=vid)
self.storeurl(playcount_url, params.originalurl,
LeComments.STEP_PALY)
if NewsStorage.getpublishdate(
params.originalurl) == TimeUtility.getintformtime(0):
if self.r.search('https?://sports\.le\.com/video/\d+\.html',
params.originalurl):
#仅针对体育频道获取发布时间
pubTime = XPathUtility(
params.content).getstring('//*[@class="live-vedio-infor"]')
publishdate = TimeUtility.getuniformtime(publishdate)
NewsStorage.setpublishdate(params.originalurl, publishdate)
else:
#仅针对综艺频道获取发布时间
title = XPathUtility(params.content).getstring(
'//h1[@class="j-video-name video-name"]')
if title:
if re.search('\d{8}', title):
publishdate = re.findall('\d{8}', title)[0]
NewsStorage.setpublishdate(params.originalurl,
publishdate)
def step3bbs(self, params):
Logger.getlogging().info("JoyComments.STEP_3")
# Step3: 通过Step2设置的url，得到所有评论，抽取评论
try:
commentsinfo = json.loads(params.content)
commentsinfo['result']['mainreplys']['rows']
except:
Logger.getlogging().warning(
'{url} Errorcode:40000'.format(url=params.originalurl))
Logger.printexception()
return
# 获取评论
for index in range(
0, int(len(commentsinfo['result']['mainreplys']['rows'])), 1):
# 提取时间
# cmti = CommentInfo()
content = commentsinfo['result']['mainreplys']['rows'][index][
'reply']['reply']['body']['text']
curtime = TimeUtility.getuniformtime(
str(commentsinfo['result']['mainreplys']['rows'][index]
['reply']['reply']['post_time']))
nick = commentsinfo['result']['mainreplys']['rows'][index][
'reply']['user']['name']
if not CMTStorage.exist(params.originalurl, content, curtime,
nick):
CMTStorage.storecmt(params.originalurl, content, curtime, nick)
def process(self, proparam):
Logger.getlogging().info(proparam.url)
try:
if proparam.step is jiemianComments.STEP_1:
# 取得url中的id
articleId = re.findall(r'^http://www\.jiemian\.com/\w+/(\d+)', proparam.url).__getitem__(0)
# 设置clicknum
self.setclick(proparam)
# 取得评论个数
comments_count = float(re.findall(r'"comment_count">(\d+)</span>', proparam.content).__getitem__(0))
if comments_count:
NewsStorage.setcmtnum(proparam.originalurl, comments_count)
# 取得评论件数
if int(comments_count) == 0:
return
# 增量判断
cmtnum = CMTStorage.getcount(proparam.originalurl, True)
if cmtnum >= comments_count:
return
page_num = int(math.ceil(float(comments_count - cmtnum) / self.PAGE_SIZE))
if page_num >= self.maxpages:
page_num = self.maxpages
# 循环取得评论的url
for page in range(1, page_num + 1, 1):
url = jiemianComments.COMMENTS_URL % (articleId, page)
self.storeurl(url, proparam.originalurl, jiemianComments.STEP_3)
elif proparam.step == jiemianComments.STEP_3:
# proparam.content = proparam.content.replace('\\','')
# soup = BeautifulSoup(proparam.content, 'html5lib')
# items = soup.select('.comment-post')
# for item in items:
# content = item.select_one('.comment-main > p').get_text().encode('utf-8')
# curtime = TimeUtility.getuniformtime(item.select_one('.date').get_text())
# nick = item.select_one('.author-name').get_text().decode('utf-8').encode('utf-8')
# 取得点赞数
votenum = self.r.getid('ding', proparam.content)
if votenum == '':
Logger.getlogging().debug("Unable to get playcount")
else:
NewsStorage.setvotenum(proparam.originalurl, votenum)
# 取得评论的正则表达式
comments = re.findall(r'<p>(.+?)<\\/p>', proparam.content)
ctime = re.findall(r'<span class=\\"date\\">(.+?)<\\/span>',proparam.content)
nicks = re.findall(r'class=\\"author-name\\">(.+?)<\\/a>', proparam.content)
# 取得评论
for index in range(0,len(comments)):
time = ctime[index].replace('\\', '')
curtime = TimeUtility.getuniformtime(time)
content = eval('u"' + comments[index] + '"').encode('utf-8')
nick = eval('u"' + nicks[index] + '"').encode('utf-8')
if not CMTStorage.exist(proparam.originalurl, content, curtime, nick):
CMTStorage.storecmt(proparam.originalurl, content, curtime, nick)
else:
Logger.getlogging().error("proparam.step == %d", proparam.step)
except Exception, e:
traceback.print_exc()
def geturlcomments(self, proparam):
# soup = BeautifulSoup(proparam.content, 'html5lib')
# lis = soup.select('.comment-say')
# for li in lis:
# content = li.select_one('.des').get_text()
# curtime = li.select_one('.time').get_text()
# nick = li.select_one('.name replyName').get_text()
# if not CMTStorage.exist(proparam.originalurl, content, curtime, nick):
# CMTStorage.storecmt(proparam.originalurl, content, curtime, nick)
# 取得评论的正则表达式
comments = re.findall(r'content":"(.+?)","paragraph_id"',
proparam.content)
commentsTime = self.r.parse(
r'origin_created":"(\d+)","member_avatarPath"', proparam.content)
nicks = self.r.parse(r'"nickname":"(.*?)","is_hot"', proparam.content)
# 取得评论
index = 0
for comment in comments:
comment = eval('u"' + comment + '"')
content = comment.encode('utf-8')
curtime = TimeUtility.getuniformtime(commentsTime[index])
nick = eval('u"' + nicks[index] + '"')
nick = nick.encode('utf-8')
if not CMTStorage.exist(proparam.originalurl, content, curtime,
nick):
CMTStorage.storecmt(proparam.originalurl, content, curtime,
nick)
index = index + 1
def geturlcomments(self, params):
xparser = XPathUtility(params.content)
# 取回所有评论
page = params.customized['page']
if page == 1:
commentstimes = xparser.getcomments(
'//table[position()>1]/tbody/tr/td/span[1]')
commentscontents = xparser.getcomments(
'//table[position()>1]/tbody/tr[2]/td[@class="post-main"]')
commentsnicks = xparser.getcomments('//*[@class="name"]/a')
else:
commentstimes = xparser.getcomments('//table/tbody/tr/td/span[1]')
commentscontents = xparser.getcomments(
'//table/tbody/tr[2]/td[@class="post-main"]')
commentsnicks = xparser.getcomments('//*[@class="name"]/a')
# 设置实际的评论量
for index in range(0, len(commentscontents), 1):
curtime = TimeUtility.getuniformtime(commentstimes[index][4:])
# 提取评论内容
content = commentscontents[index].strip()
nick = commentsnicks[index].strip()
if not CMTStorage.exist(params.originalurl, content, curtime,
nick):
CMTStorage.storecmt(params.originalurl, content, curtime, nick)
def step3news(self, params):
Logger.getlogging().info("ZolbbsComments.STEP_3")
# Step3: 通过Step2设置的url，得到所有评论，抽取评论
xparser = XPathUtility(params.content)
commentsinfo = xparser.getcomments(
'//*[@class="comment-list-new"]//*[@class="commli"]/p')
commentstime = xparser.getcomments(
'//*[@class="comment-list-new"]//*[@class="published-time"]')
commentsnick = xparser.getcomments(
'//*[@class="comment-list-new"]//*[@class="user-name"]')
# 获取评论，设置实际的评论量
for index in range(0, len(commentstime), 1):
# 提取时间
tm = commentstime[index].strip()
try:
curtime = TimeUtility.getuniformtime(getuniformtime(tm),
u'%Y-%m-%d %H:%M')
except Exception, e:
curtime = getuniformtime(tm)
# 提取评论内容
content = commentsinfo[index]
nick = commentsnick[index]
if not CMTStorage.exist(params.originalurl, content, curtime,
nick):
CMTStorage.storecmt(params.originalurl, content, curtime, nick)
def step3_ebook(self, params):
try:
jsoncontent = json.loads(params.content)
if not jsoncontent.has_key('data'):
return
html = jsoncontent['data']['listHtml']
if not html:
return
soup = BeautifulSoup(html, 'lxml')
divs = soup.select('div.cf')
if not divs:
return
for div in divs:
# commentList > dl:nth-child(1) > div.cf > dd > p:nth-child(2)
content = div.select('dd > p')[1].get_text()
curtime = TimeUtility.getuniformtime(
div.select('dd > p')[0].get_text().split('|')[-1])
nick = div.select('dd > p')[0].get_text().split('|')[0]
if not CMTStorage.exist(params.originalurl, content, curtime,
nick):
CMTStorage.storecmt(params.originalurl, content, curtime,
nick)
except Exception, e:
Logger.printexception()
def analysis(self, line, method):
try:
js = json.loads(line)
param = ProcessParam()
param.crawler_time = TimeUtility.getuniformtime(js['crawler_time'])
param.url = Common.urldec(js['foundin'])
param.content = js['html']
if method == constant.REQUEST_TYPE_POST:
param.data = js['data']
if js['html'][:3] == constant.GZIP_CODE:
param.content = zlib.decompress(param.content, 16 + zlib.MAX_WBITS)
# decode
content = Common.urldec(param.content)
charset = RegexUtility.getid('charset', content)
content = Common.trydecode(content, charset)
param.content = content
return param
except:
line = line.replace('\n', '').strip()
if not line or line[0] == '#':
return
Logger.getlogging().debug(line)
param = ProcessParam()
param.url = line
if method == constant.REQUEST_TYPE_POST:
js = json.loads(line)
param.url = js['url']
param.data = js['data']
param.content = HttpCacher.getcontent(line, method)
if param.content is None:
return
return param
def step2(self, params):
keyword = params.customized['keyword']
query = Common.urldec(keyword)
jsondata = json.loads(params.content)
# 获取分页数
html = jsondata['html']
soup = bs(html, 'html5lib')
videoUrlList = []
videoList = soup.select('li.video')
for video in videoList:
try:
videoUrl = 'https:' + video.select_one('a').get('href')
videoUrl = videoUrl.split('?')[0] + '/'
title = video.select_one('a').get('title')
pubtime = video.find(attrs={
'class': 'so-icon time'
}).get_text().strip()
if self.compareNow(TimeUtility.getuniformtime(pubtime)):
if self.checktitle(query, title):
videoUrlList.append(videoUrl)
self.__storeurl__(videoUrl, pubtime,
SPIDER_S2_WEBSITE_VIDEO)
else:
Logger.log(videoUrl,
constant.ERRORCODE_WARNNING_NOMATCHTITLE)
else:
Logger.log(videoUrl,
constant.ERRORCODE_WARNNING_NOMATCHTIME)
except:
Logger.printexception()
def dmzjnews_step3(self, params):
params.content = params.content[params.content.index('['):params.content.rindex(']') + 1]
commentsinfo = json.loads(params.content)
for index in range(0, len(commentsinfo), 1):
# 提取时间
content = commentsinfo[index]['content']
curtime = TimeUtility.getuniformtime(commentsinfo[index]['create_time'])
CMTStorage.storecmt(params.originalurl, content, curtime, '')
def process_book(self, params):
try:
if params.step == Comments.STEP_1:
# 从url中获取拼接评论url的参数
bookId = self.r.parse('^http://www\.17k\.com/book/(\w+).html$',
params.originalurl)[0]
# 拼接第一页评论url
comments_url = Comments.COMMENTS_URL % (bookId, 1,
Comments.PAGE_SIZE)
#通知下载平台，根据评论url获取第一页评论内容
self.storeurl(comments_url, params.originalurl,
Comments.STEP_2, {'bookId': bookId})
#获取第一页评论内容，循环获取全部评论url
elif params.step == Comments.STEP_2:
bookId = params.customized['bookId']
# 获取评论的Jason返回值
comments = json.loads(params.content)
comments_count = int(comments['page']['count'])
# 判断增量
cmtnum = CMTStorage.getcount(params.originalurl)
if cmtnum >= comments_count:
return
NewsStorage.setcmtnum(params.originalurl, comments_count)
# 获取评论最后更新时间
lasttime = CMTStorage.getlastpublish(params.originalurl, True)
# 获取评论页数
page_count = int(comments['page']['pagecount'])
if page_count == 0:
return
if page_count >= self.maxpages:
page_count = self.maxpages
# 循环拼接评论url，提交下载平台获取评论数据
for page in range(1, page_count + 1, 1):
commentUrl = Comments.COMMENTS_URL % (bookId, page,
Comments.PAGE_SIZE)
self.storeurl(commentUrl, params.originalurl,
Comments.STEP_3, {'bookId': bookId})
#解析评论数据
elif params.step == Comments.STEP_3:
commentsinfo = json.loads(params.content)
for comment in commentsinfo['page']['result']:
curtime = TimeUtility.getuniformtime(
comment['creationDate'])
content = comment['summary']
nick = comment['marks']['nikeName']
if not CMTStorage.exist(params.originalurl, content,
curtime, nick):
CMTStorage.storecmt(params.originalurl, content,
curtime, nick)
except Exception, e:
traceback.print_exc()
def step3(self, params):
try:
Logger.getlogging().info("Kr36Comments.STEP_3")
# Step3: 通过Step2设置的url，得到所有评论，抽取评论
jsoncontent = json.loads(params.content)
lasttime = params.customized
for index in range(0, len(jsoncontent['data']['items']), 1):
# 提取评论内容
content = jsoncontent['data']['items'][index]['content']
# 提取时间
publicTime = jsoncontent['data']['items'][index]['created_at']
curtime = TimeUtility.getuniformtime(TimeUtility.getuniformtime(publicTime, u'%Y-%m-%d %H:%M:%S'))
nick = jsoncontent['data']['items'][index]['user']['name']
if not CMTStorage.exist(params.originalurl, content, curtime, nick):
CMTStorage.storecmt(params.originalurl, content, curtime, nick)
except:
Logger.printexception()
def process(self, params):
if params.step == S2Query.STEP_1:
html = etree.HTML(params.content)
isblack = False
#判断页面是不是空的
try:
contenttext = html.xpath(
'//div[@class="attentionTitle"]/text()')
if contenttext <> []:
isblack = True
except:
isblack = False
#不是空的情况下，判断是不是又翻页
if isblack == False:
try:
page = html.xpath(
'//*[@class="pcgames_page"]/a/text()')[-2]
except:
page = '1'
index = 0
urllist = []
if self.pagelimit:
if int(page) > self.pagelimit:
Logger.getlogging().warning(
'the pageMaxNumber is shutdown to {0}'.format(
self.pagelimit))
page = self.pagelimit
for index in range(0, int(page)):
searchurl = S2Query.S2_URL % (params.customized['key'],
str(index + 1))
urllist.append(searchurl)
self.__storeqeuryurllist__(urllist, S2Query.STEP_2, {
'key': params.customized['key'],
'page': page
})
elif params.step == S2Query.STEP_2:
html = etree.HTML(params.content)
#正式环境获得数据库记载的上一次获取时候的最新评论的时间
#urlcommentinfo= URLStorage.geturlcommentinfo().updatetime
#测试环境模拟数据
urlcommentinfo = (datetime.datetime.now() + datetime.timedelta(
days=-int(self.querylastdays))).strftime('%Y-%m-%d %H:%M:%S')
urllist = html.xpath(
'//div[@class="resultList"]/ul/li/p/a[1]/@href')
pbtime = html.xpath(
'//*[@class="resultList"]/ul/li/span[1]/text()')
index = 0
url = []
for index in range(0, len(pbtime)):
pbtime[index] = pbtime[index].replace('-', '').strip()
pbtime[index] = TimeUtility.getuniformtime(
pbtime[index], '%a %b %d %H:%M:%S CST %Y')
if pbtime[index] > urlcommentinfo:
url.append(urllist[index])
if len(url) > 0:
self.__storeurllist__(url, SPIDER_S2_WEBSITE_TIEBA)
def geturlcomments(self, params, startpos=0):
# 取得所有评论
soup = BeautifulSoup(params.content, 'html5lib')
comments = soup.select('.info')
commentTimes = soup.select('.date')
commentsInfo = []
# //*[contains(@id,"postmessage_")]
if len(comments) <= 0:
tds = soup.select(
'td.plc') # soup.find_all("td", attrs={"class": "plc"})
if tds is None:
return
for td in tds:
timestr = td.find(attrs={'id': re.compile('authorposton')})
if not timestr:
continue
commentTimes = getuniformtime(timestr.get_text())
if URLStorage.storeupdatetime(params.originalurl,
commentTimes):
contents = td.find(
attrs={'id': re.compile('postmessage_')})
if contents:
cmti = CommentInfo()
cmti.content = contents.get_text()
commentsInfo.append(cmti)
else:
# 取得所有评论
for index in range(startpos, int(len(comments)), 1):
# 提取时间
cmti = CommentInfo()
publicTime = getuniformtime(
commentTimes[index].get_text()).strip()
#publicTime = self.r.parse(ur'发表于(.*)', publicTime)[0].strip()
tm = TimeUtility.getuniformtime(
TimeUtility.getuniformtime(publicTime, u'%Y-%m-%d %H:%M'))
if URLStorage.storeupdatetime(params.originalurl, tm):
cmti.content = comments[index].get_text()
commentsInfo.append(cmti)
if len(commentsInfo) > 0:
self.commentstorage.store(params.originalurl, commentsInfo)
def getcomments(self, params):
comments = json.loads(params.content)
# 获取评论
for item in comments['data']:
curtime = TimeUtility.getuniformtime(item['ctime'])
content = item['content']
nick = item['user']['username']
if not CMTStorage.exist(params.originalurl, content, curtime,
nick):
CMTStorage.storecmt(params.originalurl, content, curtime, nick)
def step_click(self, params):
sid = params.customized['sid']
infodata = json.loads(params.content)
for info in infodata:
if info['id'] == str(sid):
addtime = TimeUtility.getuniformtime(info['adddate'])
playcount = self.str2num(info['playtimes'])
NewsStorage.setclicknum(params.originalurl, playcount)
NewsStorage.setpublishdate(params.originalurl, addtime)
break
def step3(self, params):
Logger.getlogging().info("ThirtysixKryptonComments.STEP_3")
# Step3: 通过Step2设置的url，得到所有评论，抽取评论
jsoncontent = json.loads(params.content)
commentsInfo = []
for index in range(0, len(jsoncontent['data']['items']), 1):
cmti = CommentInfo()
# 提取评论内容
cmti.content = jsoncontent['data']['items'][index]['content']
# 提取时间
publicTime = jsoncontent['data']['items'][index]['created_at']
tm = TimeUtility.getuniformtime(TimeUtility.getuniformtime(publicTime, u'%Y-%m-%d %H:%M:%S'))
if URLStorage.storeupdatetime(params.originalurl, tm):
commentsInfo.append(cmti)
if len(commentsInfo) > 0:
# 保存获取的评论
self.commentstorage.store(params.originalurl, commentsInfo)
def process(self, params):
try:
if params.step is None:
# 从url中获取拼接评论url的参数
oid = self.r.parse('^http://v\.ku6\.com/show/([\w-]+..).html',
params.originalurl)[0]
# 拼接第一页评论url
comments_url = Ku6Comments.COMMENTS_URL % (oid, 1, 1)
#通知下载平台，根据评论url获取第一页评论内容
self.storeurl(comments_url, params.originalurl,
Ku6Comments.STEP_2, {'oid': oid})
#获取第一页评论内容，循环获取全部评论url
elif params.step == Ku6Comments.STEP_2:
oid = params.customized['oid']
# 获取评论的Jason返回值
comments = json.loads(params.content)
# 获取评论总数
comments_count = float(comments['data']['count'])
NewsStorage.setcmtnum(params.originalurl,
int(comments['data']['count']))
if comments_count == 0:
return
# 比较上次抓取该url的页面评论量和当前取到的评论量
cmtnum = CMTStorage.getcmtnum(params.originalurl, True)
if cmtnum >= comments_count:
return
# 循环拼接评论url，提交下载平台获取评论数据
for page in range(
0,
int(math.ceil(comments_count / Ku6Comments.PAGE_SIZE))
+ 1, 1):
commentUrl = Ku6Comments.COMMENTS_URL % (
oid, Ku6Comments.PAGE_SIZE, page + 1)
self.storeurl(commentUrl, params.originalurl,
Ku6Comments.STEP_3, {'oid': oid})
#解析评论数据
elif params.step == Ku6Comments.STEP_3:
commentsinfo = json.loads(params.content)
if not commentsinfo['data']['list']:
return
for comment in commentsinfo['data']['list']:
curtime = TimeUtility.getuniformtime(
int(comment['commentCtime']))
content = comment['commentContent']
nick = comment['commentContent']
if not CMTStorage.exist(params.originalurl, content,
curtime, nick):
CMTStorage.storecmt(params.originalurl, content,
curtime, nick)
except Exception, e:
Logger.printexception()
def step2(self, params):
"""获取评论的其他url,及评论"""
#每次spider运行的累加数据tempcmttotal
#
try:
url_id = params.customized['url_id']
comment_id = params.customized['comment_id']
before_update = params.customized['before_update']
tempcmttotal = params.customized.get('tempcmttotal', 0)
try:
jsondata = json.loads(params.content)
last = jsondata['data']['last']
hasnext = jsondata['data']['hasnext']
cmttotal = float(jsondata['data']['total'])
NewsStorage.setcmtnum(params.originalurl, cmttotal)
except:
Logger.log(params.originalurl,
constant.ERRORCODE_SITE_NOGET_COMMNETS)
return
temptimes = []
for comment in jsondata['data']['commentid']:
tempcmttotal += 1
content = comment['content']
time = TimeUtility.getuniformtime(comment['time'])
temptimes.append(time)
user = comment['userinfo'].get('nick', 'anonymous')
# 保存评论到数据库，可以通过接口exist判断评论是否已经存在
CMTStorage.storecmt(params.originalurl, content, time, user)
#对是否继续提取评论进行条件限制
nextflag = True
if temptimes:
min_update = min(temptimes)
max_update = max(temptimes)
#发布时间临界点限制:最近两天
#if max_update < self.cmtlastdays:
#nextflag = False
#发布时间限制:仅针对qq的评论提取策略，该评论的发布时间有序且依次递减
if min_update < before_update:
nextflag = False
#数量限制
if tempcmttotal >= self.comment_maxnum:
nextflag = False
if float(tempcmttotal) / self.page_size > self.maxpages:
nextflag = False
if hasnext and nextflag:
url = self.COMMENTS_URL.format(url_id, last, self.page_size)
self.storeurl(
url, params.originalurl, self.STEP_COMMENT_NEXT_PAGE, {
'url_id': url_id,
'comment_id': last,
'before_update': before_update,
'tempcmttotal': tempcmttotal
})
except:
Logger.printexception()
def step2bbs(self, params):
soup = BeautifulSoup(params.content, 'html5lib')
lis = soup.find_all(attrs={'id': re.compile('comment-\d+')})
for li in lis:
try:
curtime = li.select_one('.mh-comment-meta-date').get_text()
curtime = TimeUtility.getuniformtime(curtime)
content = li.select_one('.mh-comment-content').get_text()
CMTStorage.storecmt(params.originalurl, content, curtime, '')
except:
Logger.printexception()
def step3(self, params):
# Step3: 通过Step2设置的url，得到所有评论，抽取评论
commentsinfo = json.loads(params.content[2:-1])
contents = commentsinfo['data']['weibo']
for item in contents:
curtime = TimeUtility.getuniformtime(item['pub_time'])
content = item['content']
nick = str(item['userinfo']['nickname'])
if not CMTStorage.exist(params.originalurl, content, curtime,
nick):
CMTStorage.storecmt(params.originalurl, content, curtime, nick)
def news_step2(self, params):
"""通过评论的url获取评论"""
try:
jsondata = json.loads(params.content)
for comment in jsondata['body']:
content = str(comment['commentbody'])
nick = str(comment['commentauthor'])
curtime = TimeUtility.getuniformtime(comment['commentdate'])
if not CMTStorage.exist(params.originalurl, content, curtime, nick):
CMTStorage.storecmt(params.originalurl, content, curtime, nick)
if int(comment['reply_total']) > 0:
for index in range(0, int(comment['reply_total']), 1):
content = comment['reply'][index]['commentbody']
curtime = TimeUtility.getuniformtime(comment['reply'][index]['commentdate'])
nick = comment['reply'][index]['commentauthor']
if not CMTStorage.exist(params.originalurl, content, curtime, nick):
CMTStorage.storecmt(params.originalurl, content, curtime, nick)
except:
Logger.printexception()