Пример #1
0
    def parse(self, response):
        #获取一个选择器
        hxs = Selector(response)

        for con in hxs.xpath(self.xpathConf.get("parse_xpath")):
            ss = Selector(text=con.extract())
            el = WeiboComItemLoader(selector=ss)
            el.add_value("site_source","news.sina.com.cn")
            el.add_value("site_type","news")
            el.add_value("task_id",self.task_id)
            nowTime=time.localtime()
            nowDate=datetime.datetime(nowTime[0],nowTime[1],nowTime[2])
            el.add_value("catch_date",nowDate.strftime('%Y-%m-%d'))

            baidu_url = ss.xpath(self.xpathConf.get("site_url")).extract()
            if baidu_url and len(baidu_url)>0:
                source_url = urlUtil.getRedirectUrl(baidu_url[0],timeout=10)
                sinaUrl = fo.findSinaNewsUrl(source_url)
                if sinaUrl:
                    el.add_value('site_url',sinaUrl)
                else:
                    continue
            else:
                continue


            yield el.load_item()
    def parse(self, response):
        #获取一个选择器
        hxs = Selector(response)
        print hxs.extract()
        for con in hxs.xpath(self.xpathConf.get("parse_xpath")):

            s = Selector(text=con.extract())
            if s.xpath(self.xpathConf.get("is_parse_xpath")).__len__() > 0:
                r = [str(self.accountNum)]
                imgs = s.xpath(self.xpathConf.get("face_image")).extract()
                if len(imgs) > 0:
                    r.append(imgs[0])
                else:
                    r.append("")

                num_info = s.xpath(self.xpathConf.get("num_info")).extract()

                if len(num_info) > 0:
                    for num in fo.getWeiboCnUserInfo(num_info[0]):
                        r.append(num)
                else:
                    r.append(0)
                    r.append(0)
                    r.append(0)

                #生成保存Redis的格式
                save_info = "%s|%s|%s|%s|%s" % tuple(r)

                print(r)
                print save_info

                self.server.lpush(self.out_key, save_info)
Пример #3
0
    def parse(self, response):
        #获取一个选择器
        hxs = Selector(response)
        print hxs.extract()
        for con in hxs.xpath(self.xpathConf.get("parse_xpath")):

            s = Selector(text=con.extract())
            if s.xpath(self.xpathConf.get("is_parse_xpath")).__len__()>0:
                r = [str(self.accountNum)]
                imgs = s.xpath(self.xpathConf.get("face_image")).extract()
                if len(imgs)>0:
                    r.append(imgs[0])
                else:
                    r.append("")

                num_info = s.xpath(self.xpathConf.get("num_info")).extract()


                if len(num_info)>0:
                    for num in fo.getWeiboCnUserInfo(num_info[0]):
                        r.append(num)
                else:
                    r.append(0)
                    r.append(0)
                    r.append(0)

                #生成保存Redis的格式
                save_info = "%s|%s|%s|%s|%s" % tuple(r)

                print(r)
                print save_info



                self.server.lpush(self.out_key,save_info)
    def parse(self, response):
        #获取一个选择器
        hxs = Selector(response)

        for con in hxs.xpath(self.xpathConf.get("parse_xpath")):
            ss = Selector(text=con.extract())
            el = WeiboComItemLoader(selector=ss)
            el.add_value("site_source", "news.sina.com.cn")
            el.add_value("site_type", "news")
            el.add_value("task_id", self.task_id)
            nowTime = time.localtime()
            nowDate = datetime.datetime(nowTime[0], nowTime[1], nowTime[2])
            el.add_value("catch_date", nowDate.strftime('%Y-%m-%d'))

            baidu_url = ss.xpath(self.xpathConf.get("site_url")).extract()
            if baidu_url and len(baidu_url) > 0:
                source_url = urlUtil.getRedirectUrl(baidu_url[0], timeout=10)
                sinaUrl = fo.findSinaNewsUrl(source_url)
                if sinaUrl:
                    el.add_value('site_url', sinaUrl)
                else:
                    continue
            else:
                continue

            yield el.load_item()
Пример #5
0
    def parse(self, response):
        #获取一个选择器
        hxs = Selector(response)

        list = hxs.xpath(self.xpathConf.get("search_num"))
        print list
        if len(list) > 0:
            search_num = list[0].extract()
            num = int(fo.getBaiduSearchNum(search_num))
            if num / 10 > 80:
                self.saveUrlToRedis(80)
            else:
                self.saveUrlToRedis(num / 10)
    def parse(self, response):
        #获取一个选择器
        hxs = Selector(response)

        list = hxs.xpath(self.xpathConf.get("search_num"))

        if len(list) > 0:
            search_num = list[0].extract()
            num = int(fo.getNum(search_num))
            if num / 25 > 50:
                self.saveUrlToRedis(50)
            else:
                self.saveUrlToRedis(num / 25)
Пример #7
0
    def parse(self, response):
        #获取一个选择器
        hxs = Selector(response)

        list = hxs.xpath(self.xpathConf.get("search_num"))
        print list
        if len(list)>0:
            search_num = list[0].extract()
            num = int(fo.getBaiduSearchNum(search_num))
            if num/10>80:
                self.saveUrlToRedis(80)
            else:
                self.saveUrlToRedis(num/10)
Пример #8
0
    def parse(self, response):
        #获取一个选择器
        hxs = Selector(response)

        list = hxs.xpath(self.xpathConf.get("search_num"))

        if len(list)>0:
            search_num = list[0].extract()
            num = int(fo.getNum(search_num))
            if num/25>50:
                self.saveUrlToRedis(50)
            else:
                self.saveUrlToRedis(num/25)