Python RedisClient.get_proxy_count Examples

Example #1

class Getter(object):
    # 爬取代理网站的免费代理IP，存入redis
    def __init__(self):
        self.redis_client = RedisClient()
        self.crawler = Crawler()

    def is_full(self):
        # 判断代理池是否满了
        return self.redis_client.get_proxy_count() >= FULL_COUNT

    def run(self):
        # 将爬取到的代理存入redis
        if not self.is_full():
            proxys = self.crawler.get_crawler_proxy()
            for proxy in proxys:
                self.redis_client.add(proxy)

Example #2

File: Getter.py Project: zthaa/PythonBasic

class Getter(object):
    def __init__(self):
        self.redis_client = RedisClient()
        self.crawler = Crawler()

    def is_full(self):
        """
            判断代理池是否满了
        """
        if self.redis_client.get_proxy_count() >= FULL_COUNT:
            return True
        else:
            return False

    def run(self):
        """
            将爬取到的代理存入redis
        """
        if not self.is_full():
            proxy_list = self.crawler.get_crawler_proxy()
            for proxy in proxy_list:
                self.redis_client.add(proxy)