コード例 #1
0
	def getPostList(self,forum_name,sector_name,time_since,page_start,page_stop):
		self.forum_name=forum_name # for login purpose

		upd=None
		cnt=0
		try:
			forum=Forum.objects.filter(name=forum_name)[0:1].get()
		except Forum.DoesNotExist:
			self.logger.debug('can\'t find forum info of %s!',forum_name)
			raise

		try:
			sector=Sector.objects.filter(forum=forum,name=sector_name)[0:1].get()
		except Sector.DoesNotExist:
			self.logger.debug('can\'t find sector info of %s!',sector_name)
			raise

		for i in xrange(page_start,page_stop): # max page number
			gotnew_this_page=False # 标记本页面中是否处理了新贴/更新回帖,以便在本页没有新内容的情况下尽早退出循环

			if self.exitevent.is_set():
				self.logger.info('got exit signal!')
				break

			pageurl=sector.url%(i,)
##			self.logger.info('get %s ...',pageurl)
			data=self._getData(pageurl,None,'%s-page %02d'%(sector.name,i))
			if len(data)<100: # 论坛大姨妈了
				self.logger.info('bbs down? %s',data)
				break

			# 用 lxml xpath 选择包含正文的 div
			parser=etree.HTMLParser()
			tree=etree.fromstring(data,parser)
			el=tree.xpath(sector.url_pattern)

			for item in el: # 列表中的每个主题
				if self.exitevent.is_set():
					self.logger.info('got exit signal!')
					break
##				self.logger.debug('-='*10)
				keeptop= True if 'folderstate3' in item[0].xpath('img/@src')[0] else False # 通过图标名称判断是否为置顶贴
				if keeptop:
					gotnew_this_page=True # 避免首页都是置顶贴且无更新的情况下过早退出

				try:
					try:
						title=item[3].xpath('table/tr/td/a/text()')[0]
					except IndexError:
						try:
							title=item[3].xpath('table/tr/td/a/font/b/text()')[0]
						except IndexError:
							title=item[3].xpath('table/tr/td/a/font/text()')[0]

					# 取主题信息
					title_url=item[3].xpath('table/tr/td/a/@href')[0]
					m=re.search('TopicID=(\d+)',title_url)
					if m:
						lid=m.group(1)
					else:
						self.logger.debug('can\'t find locate_id for post %s',title)
						lid=''

					try:
						author=item[6].xpath('a/text()')[0]
					except IndexError:
						author==''
					crt_date=item[6].xpath('@title')[0][5:]
					nr_reply=int(item[5].text)

					pages= int(ceil((nr_reply)/float(self.NR_REPLY_PER_PAGE)) )

					upd_date=item[8].text.strip()
	##				self.logger.info('(%02d) (%02d) %s |%s|%s|%s|%s|%s',cnt,pages,title,author,crt_date,upd_date,nr_reply,title_url)
					cnt+=1
	##				self.logger.info('%d---%02d)%s|%s|%d-%d|%s|%s',item.sourceline,cnt,lid,upd_date,nr_reply,pages,author,title)
					self.logger.info('%s\n%02d)\t%s|%s|%d-%d|%s|%s', '-='*10, cnt, lid, upd_date, nr_reply, pages, author, title.strip())
					try:
						upd=datetime.datetime.strptime(upd_date,'%Y-%m-%d %H:%M:%S')
					except ValueError:
						self.logger.debug('upd_date=%s,no H:M:S?',upd_date)
						upd=datetime.datetime.strptime(upd_date,'%Y-%m-%d')
					if time_since and upd<time_since and (not keeptop):
						self.logger.info('%s\ngot time stop in page list.','-~'*20)
						return

					if author=='':
						author='usr for post %s'%(post.lid,)

					# check DB, insert if not exist, check update otherwise
					try:
						p=Post.objects.filter(sector=sector,locate_id=lid)[0:1].get()
					except Post.DoesNotExist:
						p=Post(sector=sector,locate_id=lid,url=title_url,title=title,author=author,
							crt_date=crt_date,upd_date=upd_date,nr_reply=0) # 先将DB中的nr_reply设为0,避免后面取帖子出错的情况下无法通过重新执行来重取帖子
						p.save()
						self.stat_post_add+=1
						p.nr_reply=nr_reply
						self.logger.debug('post %s created.',lid)
						self.getPost(p)
						gotnew_this_page=True
					else:
						if p.upd_date!=upd:
							p.upd_date=upd_date
							p.save()
						if p.nr_reply!=nr_reply:
							self.logger.info('post %s nr_reply changed. %+d',lid,nr_reply-p.nr_reply)
							self.getPost(p,nr_reply)
							gotnew_this_page=True
				except IndexError:
					if '浏览的页面或提交的数据包含敏感关键词信息,该关键词已经被过滤' in  htmlentitydecode(etree.tostring(item)):
						self.logger.info('got page contains words! %s ',htmlentitydecode(etree.tostring(item)))
						continue
					else:
						raise
コード例 #2
0
    def getPostList(self, forum_name, sector_name, time_since, page_start, page_stop):
        self.forum_name = forum_name

        upd = None
        cnt = 0
        try:
            forum = Forum.objects.filter(name=forum_name)[0:1].get()
        except Forum.DoesNotExist:
            self.logger.debug("can't find forum info of %s!", forum_name)
            raise

        try:
            sector = Sector.objects.filter(forum=forum, name=sector_name)[0:1].get()
        except Sector.DoesNotExist:
            self.logger.debug("can't find sector info of %s!", sector_name)
            raise

        for i in xrange(page_start, page_stop):  # max page number
            pageurl = sector.url % (i,)
            ##			self.logger.info('get %s ...',pageurl)
            data = self._getData(pageurl, None, "%s-page %02d" % (sector.name, i))
            if not data:
                self.logger.debug("got nothing, skip!")
                break

                # 用 lxml xpath 选择包含正文的 div
            parser = etree.HTMLParser()
            tree = etree.fromstring(data, parser)
            ##			el=tree.xpath(sector.url_pattern)
            ##			el=tree.xpath('//div[@id="threadlist"]//form[@id="moderate"]/table/*[not(@id="separatorline")]')
            el = tree.xpath('//div[@id="threadlist"]//form[@id="moderate"]/table/*[@id]')

            sticktopic = None
            for item in el:  # 列表中的每个主题
                if item.attrib["id"] == "separatorline":
                    continue

                ##				self.logger.debug('-='*10)
                if "stickthread" in item.xpath("@id")[0]:
                    sticktopic = True
                else:
                    sticktopic = False

                title = item.xpath("./tr/th/a/text()")[0]

                if item.xpath('./tr/th/span[@class="tps"]'):
                    pages = int(item.xpath('./tr/th/span[@class="tps"]/a[position()=last()]/text()')[0])
                else:
                    pages = 1

                    # 取主题信息
                ##				title_url=item.xpath('./tr/th/span[1]/a[1]/@href')[0]
                title_url = item.xpath("./tr/th/a/@href")[0]
                m = re.search("thread-(\d+)-.*?.html", title_url)
                if m:
                    lid = m.group(1)
                else:
                    self.logger.debug("can't find locate_id for post %s", title)
                    lid = ""

                author = item.xpath('./tr/td[@class="by"][1]/cite/a/text()')[0]
                assert author

                crt_date = item.xpath('./tr/td[@class="by"][1]/em/span/text()')[0]
                unknownreply = True
                try:
                    nr_reply = int(item.xpath('./tr/td[@class="num"]/a/text()')[0])
                    unknownreply = False
                except ValueError:
                    ##					self.logger.debug('post %s has no reply number!',lid) # 移动/锁贴
                    nr_reply = 0

                upd_date = item.xpath('./tr/td[@class="by"][2]/em/a/text()')[0]
                ##				self.logger.info('(%02d) (%02d) %s |%s|%s|%s|%s|%s',cnt,pages,title,author,crt_date,upd_date,nr_reply,title_url)
                cnt += 1
                ##				self.logger.info('%d---%02d)%s|%s|%d-%d|%s|%s',item.sourceline,cnt,lid,upd_date,nr_reply,pages,author,title)
                self.logger.info(
                    "%s\n%02d)\t%s|%s|%s-%d|%s|%s",
                    "-=" * 10,
                    cnt,
                    lid,
                    upd_date,
                    str(nr_reply) if not unknownreply else "?",
                    pages,
                    author,
                    title.strip(),
                )
                upd = datetime.datetime.strptime(upd_date, "%Y-%m-%d %H:%M")
                if time_since and (not sticktopic) and upd < time_since:
                    self.logger.info("%s\ngot time stop in page list.", "-~" * 20)
                    return

                    # check DB, insert if not exist, else check update
                try:
                    p = Post.objects.filter(sector=sector, locate_id=lid)[0:1].get()
                except Post.DoesNotExist:
                    p = Post(
                        sector=sector,
                        locate_id=lid,
                        url=title_url,
                        title=title,
                        author=author,
                        crt_date=crt_date,
                        upd_date=upd_date,
                        nr_reply=0,
                    )
                    p.save()
                    p.nr_reply = nr_reply
                    self.logger.debug("post %s created.", lid)
                    self.getPost(p)
                else:
                    if p.upd_date != upd:
                        p.upd_date = upd_date
                        p.save()
                    if unknownreply:  # check every time 不过每次最多只能获取一页新帖子
                        self.logger.info(
                            "post %s nr_reply may changed(unknown reply cnt). now %d in DB", lid, p.nr_reply + 1
                        )
                        nr_reply = p.nr_reply + 1  # 设为比原来的大以触发查找

                    if p.nr_reply != nr_reply:
                        self.logger.info("post %s nr_reply changed. %+d", lid, nr_reply - p.nr_reply)
                        self.getPost(p, nr_reply)
コード例 #3
0
	def getPostList(self,forum_name,sector_name,time_since,page_start,page_stop):
##		self.forum_name=forum_name
		ptoppic=re.compile('.+?t_top\d+.gif|announcement.gif'.decode(self.dft_html_encoding),re.S|re.I|re.U)
		info, debug= self.logger.info, self.logger.debug

		upd=None
		cnt=0
		try:
			forum=Forum.objects.filter(name=forum_name)[0:1].get()
		except Forum.DoesNotExist:
			debug('can\'t find forum info of %s!',forum_name)
			raise

		try:
			sector=Sector.objects.filter(forum=forum,name=sector_name)[0:1].get()
		except Sector.DoesNotExist:
			debug('can\'t find sector info of %s!',sector_name)
			raise

		for i in xrange(page_start,page_stop): # max page number
			pageurl=sector.url%(i,)
##			self.logger.info('get %s ...',pageurl)
			data=self._getData(pageurl,None,'%s-page %02d'%(sector.name,i))


			# 用 lxml xpath 选择包含正文的 div
			parser=etree.HTMLParser()
			tree=etree.fromstring(data,parser)
			el=tree.xpath('/html/body/div[@class="wrap cl"]/div[@class="main thread"]/form[@id="moderate"]/div[@class="threadlist"]/table/tbody/*')


			for item in el: # 列表中的每个主题
##				self.logger.debug('-='*10)
				if item.xpath('../@class') and item.xpath('../@class')[0]=='separation': # skip separation line
					continue

				tdlist= item.xpath('*')
				assert len(tdlist)==6
				try:
					title=tdlist[2].xpath('a/text()')[0]
				except IndexError:
					try:
						title=tdlist[2].xpath('a/font/text()')[0]
					except IndexError:
						title=tdlist[2].xpath('a/span/text()')[0]

				# 取置顶信息
				istoptopic=False
				try:
					toppic=tdlist[0].xpath('a/img/@src')[0]
				except IndexError:
					toppic=tdlist[0].xpath('img/@src')[0]
				if ptoppic.search(toppic):
					istoptopic=True

				# 取主题信息
				title_url=tdlist[2].xpath('a/@href')[0]
				m=re.search('topicid=(\d+)&?',title_url)
				if m:
					lid=m.group(1)
				else:
					debug('can\'t find locate_id for post %s',title)
					lid=''

				author=tdlist[3].xpath('cite/a/text()')[0]
				crt_date=tdlist[3].xpath('em/text()')[0]
				if crt_date.find('前')!=-1 or crt_date.find('天')!=-1:
##					debug('%s not real crt_date: %s', lid, crt_date)
					crt_date=self.str2time(crt_date).strftime('%Y-%m-%d %H:%M:%S')
				if crt_date.find('刚才')!=-1:
					crt_date=datetime.datetime.today().strftime('%Y-%m-%d %H:%M:%S')
				if crt_date.find('/')!=-1: # in form 2011/7/24 9:16:34
					crt_date=datetime.datetime.strptime(crt_date,'%Y/%m/%d %H:%M:%S').strftime('%Y-%m-%d %H:%M:%S')


				try:
					nr_reply=int(tdlist[4].xpath('a[1]/text()')[0])
				except IndexError:
					nr_reply=0

				try:
					upd_date=tdlist[5].xpath('em/a/text()')[0].strip()
				except IndexError:
					upd_date=crt_date # 找不到最近更新时间则将发表时间作为最近更新时间

				if upd_date.find('前')!=-1 or upd_date.find('天')!=-1:
##					debug('%s not real upd_date: %s', lid, upd_date)
					upd_date=self.str2time(upd_date).strftime('%Y-%m-%d %H:%M:%S')
				if upd_date.find('刚才')!=-1:
					upd_date=datetime.datetime.today().strftime('%Y-%m-%d %H:%M:%S')
				pages= int(ceil((nr_reply+1)/float(self.NR_REPLY_PER_PAGE)) )
				info('%s\n%02d)\t%s|%s|%d-%d|%s|%s', '-='*10, cnt, lid, upd_date, nr_reply, pages, author, title.strip())
				cnt+=1
				try:
					upd=datetime.datetime.strptime(upd_date,'%Y-%m-%d %H:%M:%S')
				except ValueError:
					try:
						upd=datetime.datetime.strptime(upd_date,'%Y-%m-%d %H:%M')
					except ValueError:
						upd=datetime.datetime.strptime(upd_date,'%Y/%m/%d %H:%M:%S')
#-#						upd_date=upd.strftime('%Y-%m-%d %H:%M:%S')
##				info('(%02d) (%02d) %s |%s|%s|%s|%s|%s',cnt,pages,title,author,crt_date,upd_date,nr_reply,title_url)
#-#				info('%d---%02d)%s|%s|%d-%d|%s|%s',item.sourceline,cnt,lid,upd_date,nr_reply,pages,author,title)
				if upd<time_since and (not istoptopic):
					info('%s\ngot time stop in page list.','-~'*20)
					return

				# check DB, insert if not exist, else check update
				try:
					p=Post.objects.filter(sector=sector,locate_id=lid)[0:1].get()
				except Post.DoesNotExist:
					p=Post(sector=sector, locate_id=lid, url=title_url, title=title, author=author,
						crt_date=crt_date, upd_date=upd_date, nr_reply=0)
					p.save()
					p.nr_reply=nr_reply
					debug('post %s created.',lid)
					self.getPost(p)
				else:
					if p.upd_date!=upd:
						p.upd_date=upd
						p.save()
					if p.nr_reply<nr_reply:
						info('post %s nr_reply changed. %+d',lid,nr_reply-p.nr_reply)
						self.getPost(p,nr_reply)