Python Crawler示例

示例#1

文件： getter.py 项目： TAOJIANGCAI/python3WebSpider

class Getter():
    def __init__(self):
        self.redis = RedisClient()
        self.crawler = Crawler()

    def is_over_threshold(self):
        """
        判断是否达到了代理池限制
        :return:
        """
        if self.redis.count() >= POOL_UPPER_THRESHOLD:
            return True
        else:
            return False

    def run(self):
        print('获取器开始执行')
        if not self.is_over_threshold():
            for callback_label in range(self.crawler.__CrawlFuncCount__):
                callback = self.crawler.__CrawlFunc__[callback_label]

                proxies = self.crawler.get_proxies(callback)
                sys.stdout.flush()
                for proxy in proxies:
                    print(proxy, type(proxy))
                    self.redis.add(proxy)

示例#2

class Getter():
    def __init__(self):
        self.redis = RedisClient()
        self.crawler = Crawler()

    def is_over_threshold(self):
        """
        判断数据库内容数量超过阈值
        :return: （bool）
        """
        if self.redis.count() > 10000:
            return True
        else:
            return False

    def run(self):
        """
        运行爬虫程序
        """
        print('开始获取代理')
        if not self.is_over_threshold():
            for callback_label in range(self.crawler.__CrawlFuncCount__):
                callback = self.crawler.__CrawlFunc__[callback_label]
                proxies = self.crawler.get_proxies(callback)
                for proxy in proxies:
                    self.redis.add(proxy)

示例#3

文件： main.py 项目： ganchaofan0000/ProxyPool

def addproxy():
    print('开始添加代理')
    lists = []
    Crawler().getAllproxy(lists)
    for list in lists:
        mysql.add(list[0], list[1])
    print('代理添加完成')

示例#4

文件： getter.py 项目： authetic-x/Web_Scraping

class Getter():
    def __init__(self):
        self.redis = RedisClient()
        self.crawler = Crawler()

    def is_over_threshold(self):
        if self.redis.count() >= POOL_UPPER_THRESHOLD:
            return True
        return False

    def run(self):
        print('获取器开始执行')
        if not self.is_over_threshold():
            for callback_label in range(self.crawler.__CrawlFuncCount__):
                callback = self.crawler.__crawlFunc__[callback_label]
                proxies = self.crawler.get_proxies(callback)
                for proxy in proxies:
                    self.redis.add(proxy)

示例#5

class Getter():
    def __init__(self):
        self.redis = RedisClient()
        self.crawler = Crawler()
    
    def is_over_threshold(self):
        """
        Compare to the capacity of proxy pool.
        """
        if self.redis.count() >= POOL_UPPER_THRESHOLD:
            return True
        else:
            return False
    
    def run(self):
        logger.debug('Getter is running.')
        if not self.is_over_threshold():
            for callback_label in range(self.crawler.__CrawlFuncCount__):
                callback = self.crawler.__CrawlFunc__[callback_label]
                # 获取代理
                proxies = self.crawler.get_proxies(callback)
                sys.stdout.flush()
                for proxy in proxies:
                    self.redis.add(proxy)

示例#6

文件： getter.py 项目： authetic-x/Web_Scraping

 def __init__(self):
     self.redis = RedisClient()
     self.crawler = Crawler()

示例#7

# -*- coding: utf-8 -*-
from ProxyPool.crawler import Crawler

c = Crawler()
proxies = c.crawl_89ip()
for proxy in proxies:
    print(proxy)